@ohos-ports/confluentinc-kafka-javascript 1.10.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (628) hide show
  1. package/CHANGELOG.md +457 -0
  2. package/LICENSE.kafkajs +31 -0
  3. package/LICENSE.node-rdkafka +25 -0
  4. package/LICENSE.txt +20 -0
  5. package/README.md +133 -0
  6. package/binding.gyp +184 -0
  7. package/build/Release/confluent-kafka-javascript.node +0 -0
  8. package/build/deps/librdkafka++.so.1 +0 -0
  9. package/build/deps/librdkafka.Makefile +6 -0
  10. package/build/deps/librdkafka.so.1 +0 -0
  11. package/build/deps/librdkafka.target.mk +55 -0
  12. package/deps/librdkafka/.clang-format +136 -0
  13. package/deps/librdkafka/.clang-format-cpp +103 -0
  14. package/deps/librdkafka/.dir-locals.el +10 -0
  15. package/deps/librdkafka/.formatignore +35 -0
  16. package/deps/librdkafka/.gdbmacros +19 -0
  17. package/deps/librdkafka/CHANGELOG.md +2477 -0
  18. package/deps/librdkafka/CMakeLists.txt +291 -0
  19. package/deps/librdkafka/CODE_OF_CONDUCT.md +46 -0
  20. package/deps/librdkafka/CONFIGURATION.md +212 -0
  21. package/deps/librdkafka/CONTRIBUTING.md +431 -0
  22. package/deps/librdkafka/Doxyfile +2375 -0
  23. package/deps/librdkafka/INTRODUCTION.md +2752 -0
  24. package/deps/librdkafka/LICENSE +26 -0
  25. package/deps/librdkafka/LICENSE.cjson +22 -0
  26. package/deps/librdkafka/LICENSE.crc32c +28 -0
  27. package/deps/librdkafka/LICENSE.fnv1a +18 -0
  28. package/deps/librdkafka/LICENSE.hdrhistogram +27 -0
  29. package/deps/librdkafka/LICENSE.lz4 +26 -0
  30. package/deps/librdkafka/LICENSE.murmur2 +25 -0
  31. package/deps/librdkafka/LICENSE.nanopb +22 -0
  32. package/deps/librdkafka/LICENSE.opentelemetry +203 -0
  33. package/deps/librdkafka/LICENSE.pycrc +23 -0
  34. package/deps/librdkafka/LICENSE.queue +31 -0
  35. package/deps/librdkafka/LICENSE.regexp +5 -0
  36. package/deps/librdkafka/LICENSE.snappy +36 -0
  37. package/deps/librdkafka/LICENSE.tinycthread +26 -0
  38. package/deps/librdkafka/LICENSE.wingetopt +49 -0
  39. package/deps/librdkafka/LICENSES.txt +625 -0
  40. package/deps/librdkafka/Makefile +125 -0
  41. package/deps/librdkafka/README.md +201 -0
  42. package/deps/librdkafka/README.win32 +26 -0
  43. package/deps/librdkafka/STATISTICS.md +624 -0
  44. package/deps/librdkafka/configure +214 -0
  45. package/deps/librdkafka/configure.self +341 -0
  46. package/deps/librdkafka/dev-conf.sh +123 -0
  47. package/deps/librdkafka/lds-gen.py +73 -0
  48. package/deps/librdkafka/mainpage.doxy +40 -0
  49. package/deps/librdkafka/mklove/Makefile.base +329 -0
  50. package/deps/librdkafka/mklove/modules/configure.atomics +144 -0
  51. package/deps/librdkafka/mklove/modules/configure.base +2484 -0
  52. package/deps/librdkafka/mklove/modules/configure.builtin +70 -0
  53. package/deps/librdkafka/mklove/modules/configure.cc +186 -0
  54. package/deps/librdkafka/mklove/modules/configure.cxx +8 -0
  55. package/deps/librdkafka/mklove/modules/configure.fileversion +65 -0
  56. package/deps/librdkafka/mklove/modules/configure.gitversion +29 -0
  57. package/deps/librdkafka/mklove/modules/configure.good_cflags +18 -0
  58. package/deps/librdkafka/mklove/modules/configure.host +132 -0
  59. package/deps/librdkafka/mklove/modules/configure.lib +49 -0
  60. package/deps/librdkafka/mklove/modules/configure.libcurl +99 -0
  61. package/deps/librdkafka/mklove/modules/configure.libsasl2 +36 -0
  62. package/deps/librdkafka/mklove/modules/configure.libssl +147 -0
  63. package/deps/librdkafka/mklove/modules/configure.libzstd +58 -0
  64. package/deps/librdkafka/mklove/modules/configure.parseversion +95 -0
  65. package/deps/librdkafka/mklove/modules/configure.pic +16 -0
  66. package/deps/librdkafka/mklove/modules/configure.socket +20 -0
  67. package/deps/librdkafka/mklove/modules/configure.zlib +61 -0
  68. package/deps/librdkafka/mklove/modules/patches/README.md +8 -0
  69. package/deps/librdkafka/mklove/modules/patches/libcurl.0000-no-runtime-linking-check.patch +11 -0
  70. package/deps/librdkafka/mklove/modules/patches/libssl.0000-osx-rand-include-fix-OpenSSL-PR16409.patch +56 -0
  71. package/deps/librdkafka/service.yml +182 -0
  72. package/deps/librdkafka/src/CMakeLists.txt +379 -0
  73. package/deps/librdkafka/src/Makefile +105 -0
  74. package/deps/librdkafka/src/README.lz4.md +30 -0
  75. package/deps/librdkafka/src/cJSON.c +3191 -0
  76. package/deps/librdkafka/src/cJSON.h +306 -0
  77. package/deps/librdkafka/src/crc32c.c +430 -0
  78. package/deps/librdkafka/src/crc32c.h +38 -0
  79. package/deps/librdkafka/src/generate_proto.sh +66 -0
  80. package/deps/librdkafka/src/librdkafka_cgrp_synch.png +0 -0
  81. package/deps/librdkafka/src/lz4.c +2727 -0
  82. package/deps/librdkafka/src/lz4.h +842 -0
  83. package/deps/librdkafka/src/lz4frame.c +2078 -0
  84. package/deps/librdkafka/src/lz4frame.h +692 -0
  85. package/deps/librdkafka/src/lz4frame_static.h +47 -0
  86. package/deps/librdkafka/src/lz4hc.c +1631 -0
  87. package/deps/librdkafka/src/lz4hc.h +413 -0
  88. package/deps/librdkafka/src/nanopb/pb.h +917 -0
  89. package/deps/librdkafka/src/nanopb/pb_common.c +388 -0
  90. package/deps/librdkafka/src/nanopb/pb_common.h +49 -0
  91. package/deps/librdkafka/src/nanopb/pb_decode.c +1727 -0
  92. package/deps/librdkafka/src/nanopb/pb_decode.h +193 -0
  93. package/deps/librdkafka/src/nanopb/pb_encode.c +1000 -0
  94. package/deps/librdkafka/src/nanopb/pb_encode.h +185 -0
  95. package/deps/librdkafka/src/opentelemetry/common.pb.c +32 -0
  96. package/deps/librdkafka/src/opentelemetry/common.pb.h +170 -0
  97. package/deps/librdkafka/src/opentelemetry/metrics.options +2 -0
  98. package/deps/librdkafka/src/opentelemetry/metrics.pb.c +67 -0
  99. package/deps/librdkafka/src/opentelemetry/metrics.pb.h +966 -0
  100. package/deps/librdkafka/src/opentelemetry/resource.pb.c +12 -0
  101. package/deps/librdkafka/src/opentelemetry/resource.pb.h +58 -0
  102. package/deps/librdkafka/src/queue.h +850 -0
  103. package/deps/librdkafka/src/rd.h +592 -0
  104. package/deps/librdkafka/src/rdaddr.c +255 -0
  105. package/deps/librdkafka/src/rdaddr.h +202 -0
  106. package/deps/librdkafka/src/rdatomic.h +262 -0
  107. package/deps/librdkafka/src/rdavg.h +260 -0
  108. package/deps/librdkafka/src/rdavl.c +210 -0
  109. package/deps/librdkafka/src/rdavl.h +250 -0
  110. package/deps/librdkafka/src/rdbase64.c +200 -0
  111. package/deps/librdkafka/src/rdbase64.h +43 -0
  112. package/deps/librdkafka/src/rdbuf.c +1884 -0
  113. package/deps/librdkafka/src/rdbuf.h +375 -0
  114. package/deps/librdkafka/src/rdcrc32.c +114 -0
  115. package/deps/librdkafka/src/rdcrc32.h +170 -0
  116. package/deps/librdkafka/src/rddl.c +179 -0
  117. package/deps/librdkafka/src/rddl.h +43 -0
  118. package/deps/librdkafka/src/rdendian.h +175 -0
  119. package/deps/librdkafka/src/rdfloat.h +67 -0
  120. package/deps/librdkafka/src/rdfnv1a.c +113 -0
  121. package/deps/librdkafka/src/rdfnv1a.h +35 -0
  122. package/deps/librdkafka/src/rdgz.c +120 -0
  123. package/deps/librdkafka/src/rdgz.h +46 -0
  124. package/deps/librdkafka/src/rdhdrhistogram.c +721 -0
  125. package/deps/librdkafka/src/rdhdrhistogram.h +87 -0
  126. package/deps/librdkafka/src/rdhttp.c +831 -0
  127. package/deps/librdkafka/src/rdhttp.h +101 -0
  128. package/deps/librdkafka/src/rdinterval.h +177 -0
  129. package/deps/librdkafka/src/rdkafka.c +7287 -0
  130. package/deps/librdkafka/src/rdkafka.h +11795 -0
  131. package/deps/librdkafka/src/rdkafka_admin.c +10165 -0
  132. package/deps/librdkafka/src/rdkafka_admin.h +658 -0
  133. package/deps/librdkafka/src/rdkafka_assignment.c +1282 -0
  134. package/deps/librdkafka/src/rdkafka_assignment.h +73 -0
  135. package/deps/librdkafka/src/rdkafka_assignor.c +1800 -0
  136. package/deps/librdkafka/src/rdkafka_assignor.h +402 -0
  137. package/deps/librdkafka/src/rdkafka_aux.c +409 -0
  138. package/deps/librdkafka/src/rdkafka_aux.h +174 -0
  139. package/deps/librdkafka/src/rdkafka_background.c +222 -0
  140. package/deps/librdkafka/src/rdkafka_broker.c +6736 -0
  141. package/deps/librdkafka/src/rdkafka_broker.h +851 -0
  142. package/deps/librdkafka/src/rdkafka_buf.c +543 -0
  143. package/deps/librdkafka/src/rdkafka_buf.h +1525 -0
  144. package/deps/librdkafka/src/rdkafka_cert.c +576 -0
  145. package/deps/librdkafka/src/rdkafka_cert.h +62 -0
  146. package/deps/librdkafka/src/rdkafka_cgrp.c +8310 -0
  147. package/deps/librdkafka/src/rdkafka_cgrp.h +589 -0
  148. package/deps/librdkafka/src/rdkafka_conf.c +5177 -0
  149. package/deps/librdkafka/src/rdkafka_conf.h +746 -0
  150. package/deps/librdkafka/src/rdkafka_confval.h +97 -0
  151. package/deps/librdkafka/src/rdkafka_coord.c +623 -0
  152. package/deps/librdkafka/src/rdkafka_coord.h +132 -0
  153. package/deps/librdkafka/src/rdkafka_error.c +228 -0
  154. package/deps/librdkafka/src/rdkafka_error.h +80 -0
  155. package/deps/librdkafka/src/rdkafka_event.c +502 -0
  156. package/deps/librdkafka/src/rdkafka_event.h +126 -0
  157. package/deps/librdkafka/src/rdkafka_feature.c +898 -0
  158. package/deps/librdkafka/src/rdkafka_feature.h +104 -0
  159. package/deps/librdkafka/src/rdkafka_fetcher.c +3820 -0
  160. package/deps/librdkafka/src/rdkafka_fetcher.h +62 -0
  161. package/deps/librdkafka/src/rdkafka_header.c +220 -0
  162. package/deps/librdkafka/src/rdkafka_header.h +76 -0
  163. package/deps/librdkafka/src/rdkafka_idempotence.c +807 -0
  164. package/deps/librdkafka/src/rdkafka_idempotence.h +144 -0
  165. package/deps/librdkafka/src/rdkafka_int.h +1505 -0
  166. package/deps/librdkafka/src/rdkafka_interceptor.c +842 -0
  167. package/deps/librdkafka/src/rdkafka_interceptor.h +104 -0
  168. package/deps/librdkafka/src/rdkafka_lz4.c +450 -0
  169. package/deps/librdkafka/src/rdkafka_lz4.h +49 -0
  170. package/deps/librdkafka/src/rdkafka_metadata.c +2347 -0
  171. package/deps/librdkafka/src/rdkafka_metadata.h +350 -0
  172. package/deps/librdkafka/src/rdkafka_metadata_cache.c +1183 -0
  173. package/deps/librdkafka/src/rdkafka_mock.c +4514 -0
  174. package/deps/librdkafka/src/rdkafka_mock.h +867 -0
  175. package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1877 -0
  176. package/deps/librdkafka/src/rdkafka_mock_handlers.c +5255 -0
  177. package/deps/librdkafka/src/rdkafka_mock_int.h +1005 -0
  178. package/deps/librdkafka/src/rdkafka_mock_sharegrp.c +1207 -0
  179. package/deps/librdkafka/src/rdkafka_msg.c +2619 -0
  180. package/deps/librdkafka/src/rdkafka_msg.h +629 -0
  181. package/deps/librdkafka/src/rdkafka_msgbatch.h +62 -0
  182. package/deps/librdkafka/src/rdkafka_msgset.h +105 -0
  183. package/deps/librdkafka/src/rdkafka_msgset_reader.c +2099 -0
  184. package/deps/librdkafka/src/rdkafka_msgset_writer.c +1474 -0
  185. package/deps/librdkafka/src/rdkafka_offset.c +1565 -0
  186. package/deps/librdkafka/src/rdkafka_offset.h +150 -0
  187. package/deps/librdkafka/src/rdkafka_op.c +1219 -0
  188. package/deps/librdkafka/src/rdkafka_op.h +1065 -0
  189. package/deps/librdkafka/src/rdkafka_partition.c +5109 -0
  190. package/deps/librdkafka/src/rdkafka_partition.h +1215 -0
  191. package/deps/librdkafka/src/rdkafka_pattern.c +228 -0
  192. package/deps/librdkafka/src/rdkafka_pattern.h +70 -0
  193. package/deps/librdkafka/src/rdkafka_plugin.c +213 -0
  194. package/deps/librdkafka/src/rdkafka_plugin.h +41 -0
  195. package/deps/librdkafka/src/rdkafka_proto.h +739 -0
  196. package/deps/librdkafka/src/rdkafka_protocol.h +131 -0
  197. package/deps/librdkafka/src/rdkafka_queue.c +1351 -0
  198. package/deps/librdkafka/src/rdkafka_queue.h +1235 -0
  199. package/deps/librdkafka/src/rdkafka_range_assignor.c +1748 -0
  200. package/deps/librdkafka/src/rdkafka_request.c +7298 -0
  201. package/deps/librdkafka/src/rdkafka_request.h +761 -0
  202. package/deps/librdkafka/src/rdkafka_roundrobin_assignor.c +123 -0
  203. package/deps/librdkafka/src/rdkafka_sasl.c +589 -0
  204. package/deps/librdkafka/src/rdkafka_sasl.h +63 -0
  205. package/deps/librdkafka/src/rdkafka_sasl_cyrus.c +722 -0
  206. package/deps/librdkafka/src/rdkafka_sasl_int.h +89 -0
  207. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +1833 -0
  208. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.h +52 -0
  209. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1827 -0
  210. package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +52 -0
  211. package/deps/librdkafka/src/rdkafka_sasl_plain.c +142 -0
  212. package/deps/librdkafka/src/rdkafka_sasl_scram.c +858 -0
  213. package/deps/librdkafka/src/rdkafka_sasl_win32.c +550 -0
  214. package/deps/librdkafka/src/rdkafka_share_acknowledgement.c +1445 -0
  215. package/deps/librdkafka/src/rdkafka_share_acknowledgement.h +373 -0
  216. package/deps/librdkafka/src/rdkafka_ssl.c +2235 -0
  217. package/deps/librdkafka/src/rdkafka_ssl.h +86 -0
  218. package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4785 -0
  219. package/deps/librdkafka/src/rdkafka_subscription.c +411 -0
  220. package/deps/librdkafka/src/rdkafka_telemetry.c +761 -0
  221. package/deps/librdkafka/src/rdkafka_telemetry.h +52 -0
  222. package/deps/librdkafka/src/rdkafka_telemetry_decode.c +1307 -0
  223. package/deps/librdkafka/src/rdkafka_telemetry_decode.h +59 -0
  224. package/deps/librdkafka/src/rdkafka_telemetry_encode.c +1309 -0
  225. package/deps/librdkafka/src/rdkafka_telemetry_encode.h +459 -0
  226. package/deps/librdkafka/src/rdkafka_timer.c +407 -0
  227. package/deps/librdkafka/src/rdkafka_timer.h +117 -0
  228. package/deps/librdkafka/src/rdkafka_topic.c +2382 -0
  229. package/deps/librdkafka/src/rdkafka_topic.h +369 -0
  230. package/deps/librdkafka/src/rdkafka_transport.c +1309 -0
  231. package/deps/librdkafka/src/rdkafka_transport.h +99 -0
  232. package/deps/librdkafka/src/rdkafka_transport_int.h +100 -0
  233. package/deps/librdkafka/src/rdkafka_txnmgr.c +3256 -0
  234. package/deps/librdkafka/src/rdkafka_txnmgr.h +171 -0
  235. package/deps/librdkafka/src/rdkafka_zstd.c +226 -0
  236. package/deps/librdkafka/src/rdkafka_zstd.h +57 -0
  237. package/deps/librdkafka/src/rdlist.c +593 -0
  238. package/deps/librdkafka/src/rdlist.h +444 -0
  239. package/deps/librdkafka/src/rdlog.c +89 -0
  240. package/deps/librdkafka/src/rdlog.h +41 -0
  241. package/deps/librdkafka/src/rdmap.c +508 -0
  242. package/deps/librdkafka/src/rdmap.h +492 -0
  243. package/deps/librdkafka/src/rdmurmur2.c +167 -0
  244. package/deps/librdkafka/src/rdmurmur2.h +35 -0
  245. package/deps/librdkafka/src/rdports.c +61 -0
  246. package/deps/librdkafka/src/rdports.h +38 -0
  247. package/deps/librdkafka/src/rdposix.h +250 -0
  248. package/deps/librdkafka/src/rdrand.c +151 -0
  249. package/deps/librdkafka/src/rdrand.h +72 -0
  250. package/deps/librdkafka/src/rdregex.c +156 -0
  251. package/deps/librdkafka/src/rdregex.h +43 -0
  252. package/deps/librdkafka/src/rdsignal.h +57 -0
  253. package/deps/librdkafka/src/rdstring.c +645 -0
  254. package/deps/librdkafka/src/rdstring.h +98 -0
  255. package/deps/librdkafka/src/rdsysqueue.h +404 -0
  256. package/deps/librdkafka/src/rdtime.h +356 -0
  257. package/deps/librdkafka/src/rdtypes.h +86 -0
  258. package/deps/librdkafka/src/rdunittest.c +564 -0
  259. package/deps/librdkafka/src/rdunittest.h +232 -0
  260. package/deps/librdkafka/src/rdunittest_acknowledge.c +1177 -0
  261. package/deps/librdkafka/src/rdunittest_fetcher.c +1211 -0
  262. package/deps/librdkafka/src/rdunittest_msgset_errors.c +1521 -0
  263. package/deps/librdkafka/src/rdvarint.c +134 -0
  264. package/deps/librdkafka/src/rdvarint.h +165 -0
  265. package/deps/librdkafka/src/rdwin32.h +382 -0
  266. package/deps/librdkafka/src/rdxxhash.c +1030 -0
  267. package/deps/librdkafka/src/rdxxhash.h +328 -0
  268. package/deps/librdkafka/src/regexp.c +1352 -0
  269. package/deps/librdkafka/src/regexp.h +41 -0
  270. package/deps/librdkafka/src/snappy.c +1866 -0
  271. package/deps/librdkafka/src/snappy.h +62 -0
  272. package/deps/librdkafka/src/snappy_compat.h +138 -0
  273. package/deps/librdkafka/src/statistics_schema.json +444 -0
  274. package/deps/librdkafka/src/tinycthread.c +932 -0
  275. package/deps/librdkafka/src/tinycthread.h +503 -0
  276. package/deps/librdkafka/src/tinycthread_extra.c +210 -0
  277. package/deps/librdkafka/src/tinycthread_extra.h +218 -0
  278. package/deps/librdkafka/src/win32_config.h +58 -0
  279. package/deps/librdkafka/src-cpp/CMakeLists.txt +90 -0
  280. package/deps/librdkafka/src-cpp/ConfImpl.cpp +84 -0
  281. package/deps/librdkafka/src-cpp/ConsumerImpl.cpp +244 -0
  282. package/deps/librdkafka/src-cpp/HandleImpl.cpp +436 -0
  283. package/deps/librdkafka/src-cpp/HeadersImpl.cpp +48 -0
  284. package/deps/librdkafka/src-cpp/KafkaConsumerImpl.cpp +296 -0
  285. package/deps/librdkafka/src-cpp/Makefile +55 -0
  286. package/deps/librdkafka/src-cpp/MessageImpl.cpp +38 -0
  287. package/deps/librdkafka/src-cpp/MetadataImpl.cpp +170 -0
  288. package/deps/librdkafka/src-cpp/ProducerImpl.cpp +197 -0
  289. package/deps/librdkafka/src-cpp/QueueImpl.cpp +70 -0
  290. package/deps/librdkafka/src-cpp/README.md +16 -0
  291. package/deps/librdkafka/src-cpp/RdKafka.cpp +59 -0
  292. package/deps/librdkafka/src-cpp/TopicImpl.cpp +124 -0
  293. package/deps/librdkafka/src-cpp/TopicPartitionImpl.cpp +57 -0
  294. package/deps/librdkafka/src-cpp/rdkafkacpp.h +3797 -0
  295. package/deps/librdkafka/src-cpp/rdkafkacpp_int.h +1641 -0
  296. package/deps/librdkafka/tests/0000-unittests.c +72 -0
  297. package/deps/librdkafka/tests/0001-multiobj.c +102 -0
  298. package/deps/librdkafka/tests/0002-unkpart.c +244 -0
  299. package/deps/librdkafka/tests/0003-msgmaxsize.c +173 -0
  300. package/deps/librdkafka/tests/0004-conf.c +934 -0
  301. package/deps/librdkafka/tests/0005-order.c +133 -0
  302. package/deps/librdkafka/tests/0006-symbols.c +163 -0
  303. package/deps/librdkafka/tests/0007-autotopic.c +136 -0
  304. package/deps/librdkafka/tests/0008-reqacks.c +179 -0
  305. package/deps/librdkafka/tests/0009-mock_cluster.c +272 -0
  306. package/deps/librdkafka/tests/0011-produce_batch.c +753 -0
  307. package/deps/librdkafka/tests/0012-produce_consume.c +537 -0
  308. package/deps/librdkafka/tests/0013-null-msgs.c +473 -0
  309. package/deps/librdkafka/tests/0014-reconsume-191.c +512 -0
  310. package/deps/librdkafka/tests/0015-offset_seeks.c +172 -0
  311. package/deps/librdkafka/tests/0016-client_swname.c +181 -0
  312. package/deps/librdkafka/tests/0017-compression.c +140 -0
  313. package/deps/librdkafka/tests/0018-cgrp_term.c +444 -0
  314. package/deps/librdkafka/tests/0019-list_groups.c +289 -0
  315. package/deps/librdkafka/tests/0020-destroy_hang.c +162 -0
  316. package/deps/librdkafka/tests/0021-rkt_destroy.c +72 -0
  317. package/deps/librdkafka/tests/0022-consume_batch.c +331 -0
  318. package/deps/librdkafka/tests/0025-timers.c +147 -0
  319. package/deps/librdkafka/tests/0026-consume_pause.c +547 -0
  320. package/deps/librdkafka/tests/0028-long_topicnames.c +79 -0
  321. package/deps/librdkafka/tests/0029-assign_offset.c +202 -0
  322. package/deps/librdkafka/tests/0030-offset_commit.c +589 -0
  323. package/deps/librdkafka/tests/0031-get_offsets.c +235 -0
  324. package/deps/librdkafka/tests/0033-regex_subscribe.c +536 -0
  325. package/deps/librdkafka/tests/0034-offset_reset.c +398 -0
  326. package/deps/librdkafka/tests/0035-api_version.c +73 -0
  327. package/deps/librdkafka/tests/0036-partial_fetch.c +87 -0
  328. package/deps/librdkafka/tests/0037-destroy_hang_local.c +85 -0
  329. package/deps/librdkafka/tests/0038-performance.c +121 -0
  330. package/deps/librdkafka/tests/0039-event.c +284 -0
  331. package/deps/librdkafka/tests/0040-io_event.c +257 -0
  332. package/deps/librdkafka/tests/0041-fetch_max_bytes.c +97 -0
  333. package/deps/librdkafka/tests/0042-many_topics.c +252 -0
  334. package/deps/librdkafka/tests/0043-no_connection.c +77 -0
  335. package/deps/librdkafka/tests/0044-partition_cnt.c +94 -0
  336. package/deps/librdkafka/tests/0045-subscribe_update.c +1010 -0
  337. package/deps/librdkafka/tests/0046-rkt_cache.c +65 -0
  338. package/deps/librdkafka/tests/0047-partial_buf_tmout.c +98 -0
  339. package/deps/librdkafka/tests/0048-partitioner.c +283 -0
  340. package/deps/librdkafka/tests/0049-consume_conn_close.c +162 -0
  341. package/deps/librdkafka/tests/0050-subscribe_adds.c +145 -0
  342. package/deps/librdkafka/tests/0051-assign_adds.c +126 -0
  343. package/deps/librdkafka/tests/0052-msg_timestamps.c +238 -0
  344. package/deps/librdkafka/tests/0053-stats_cb.cpp +527 -0
  345. package/deps/librdkafka/tests/0054-offset_time.cpp +236 -0
  346. package/deps/librdkafka/tests/0055-producer_latency.c +539 -0
  347. package/deps/librdkafka/tests/0056-balanced_group_mt.c +315 -0
  348. package/deps/librdkafka/tests/0057-invalid_topic.cpp +112 -0
  349. package/deps/librdkafka/tests/0058-log.cpp +123 -0
  350. package/deps/librdkafka/tests/0059-bsearch.cpp +241 -0
  351. package/deps/librdkafka/tests/0060-op_prio.cpp +163 -0
  352. package/deps/librdkafka/tests/0061-consumer_lag.cpp +295 -0
  353. package/deps/librdkafka/tests/0062-stats_event.c +126 -0
  354. package/deps/librdkafka/tests/0063-clusterid.cpp +180 -0
  355. package/deps/librdkafka/tests/0064-interceptors.c +529 -0
  356. package/deps/librdkafka/tests/0065-yield.cpp +140 -0
  357. package/deps/librdkafka/tests/0066-plugins.cpp +143 -0
  358. package/deps/librdkafka/tests/0067-empty_topic.cpp +151 -0
  359. package/deps/librdkafka/tests/0068-produce_timeout.c +136 -0
  360. package/deps/librdkafka/tests/0069-consumer_add_parts.c +119 -0
  361. package/deps/librdkafka/tests/0070-null_empty.cpp +197 -0
  362. package/deps/librdkafka/tests/0072-headers_ut.c +448 -0
  363. package/deps/librdkafka/tests/0073-headers.c +381 -0
  364. package/deps/librdkafka/tests/0074-producev.c +87 -0
  365. package/deps/librdkafka/tests/0075-retry.c +290 -0
  366. package/deps/librdkafka/tests/0076-produce_retry.c +452 -0
  367. package/deps/librdkafka/tests/0077-compaction.c +366 -0
  368. package/deps/librdkafka/tests/0078-c_from_cpp.cpp +96 -0
  369. package/deps/librdkafka/tests/0079-fork.c +93 -0
  370. package/deps/librdkafka/tests/0080-admin_ut.c +3095 -0
  371. package/deps/librdkafka/tests/0081-admin.c +5633 -0
  372. package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +137 -0
  373. package/deps/librdkafka/tests/0083-cb_event.c +233 -0
  374. package/deps/librdkafka/tests/0084-destroy_flags.c +208 -0
  375. package/deps/librdkafka/tests/0085-headers.cpp +392 -0
  376. package/deps/librdkafka/tests/0086-purge.c +368 -0
  377. package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +162 -0
  378. package/deps/librdkafka/tests/0089-max_poll_interval.c +511 -0
  379. package/deps/librdkafka/tests/0090-idempotence.c +171 -0
  380. package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +295 -0
  381. package/deps/librdkafka/tests/0092-mixed_msgver.c +103 -0
  382. package/deps/librdkafka/tests/0093-holb.c +200 -0
  383. package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +231 -0
  384. package/deps/librdkafka/tests/0095-all_brokers_down.cpp +122 -0
  385. package/deps/librdkafka/tests/0097-ssl_verify.cpp +674 -0
  386. package/deps/librdkafka/tests/0098-consumer-txn.cpp +1218 -0
  387. package/deps/librdkafka/tests/0099-commit_metadata.c +194 -0
  388. package/deps/librdkafka/tests/0100-thread_interceptors.cpp +195 -0
  389. package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +446 -0
  390. package/deps/librdkafka/tests/0102-static_group_rebalance.c +836 -0
  391. package/deps/librdkafka/tests/0103-transactions.c +1383 -0
  392. package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +625 -0
  393. package/deps/librdkafka/tests/0105-transactions_mock.c +3930 -0
  394. package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +318 -0
  395. package/deps/librdkafka/tests/0107-topic_recreate.c +259 -0
  396. package/deps/librdkafka/tests/0109-auto_create_topics.cpp +278 -0
  397. package/deps/librdkafka/tests/0110-batch_size.cpp +182 -0
  398. package/deps/librdkafka/tests/0111-delay_create_topics.cpp +127 -0
  399. package/deps/librdkafka/tests/0112-assign_unknown_part.c +87 -0
  400. package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +3473 -0
  401. package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +176 -0
  402. package/deps/librdkafka/tests/0115-producer_auth.cpp +182 -0
  403. package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +216 -0
  404. package/deps/librdkafka/tests/0117-mock_errors.c +331 -0
  405. package/deps/librdkafka/tests/0118-commit_rebalance.c +154 -0
  406. package/deps/librdkafka/tests/0119-consumer_auth.cpp +167 -0
  407. package/deps/librdkafka/tests/0120-asymmetric_subscription.c +185 -0
  408. package/deps/librdkafka/tests/0121-clusterid.c +115 -0
  409. package/deps/librdkafka/tests/0122-buffer_cleaning_after_rebalance.c +227 -0
  410. package/deps/librdkafka/tests/0123-connections_max_idle.c +98 -0
  411. package/deps/librdkafka/tests/0124-openssl_invalid_engine.c +69 -0
  412. package/deps/librdkafka/tests/0125-immediate_flush.c +144 -0
  413. package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +1164 -0
  414. package/deps/librdkafka/tests/0127-fetch_queue_backoff.cpp +165 -0
  415. package/deps/librdkafka/tests/0128-sasl_callback_queue.cpp +217 -0
  416. package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +79 -0
  417. package/deps/librdkafka/tests/0130-store_offsets.c +178 -0
  418. package/deps/librdkafka/tests/0131-connect_timeout.c +81 -0
  419. package/deps/librdkafka/tests/0132-strategy_ordering.c +179 -0
  420. package/deps/librdkafka/tests/0133-ssl_keys.c +150 -0
  421. package/deps/librdkafka/tests/0134-ssl_provider.c +92 -0
  422. package/deps/librdkafka/tests/0135-sasl_credentials.cpp +279 -0
  423. package/deps/librdkafka/tests/0136-resolve_cb.c +181 -0
  424. package/deps/librdkafka/tests/0137-barrier_batch_consume.c +752 -0
  425. package/deps/librdkafka/tests/0138-admin_mock.c +281 -0
  426. package/deps/librdkafka/tests/0139-offset_validation_mock.c +950 -0
  427. package/deps/librdkafka/tests/0140-commit_metadata.cpp +108 -0
  428. package/deps/librdkafka/tests/0142-reauthentication.c +931 -0
  429. package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +552 -0
  430. package/deps/librdkafka/tests/0144-idempotence_mock.c +373 -0
  431. package/deps/librdkafka/tests/0145-pause_resume_mock.c +119 -0
  432. package/deps/librdkafka/tests/0146-metadata_mock.c +505 -0
  433. package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +1210 -0
  434. package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +569 -0
  435. package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
  436. package/deps/librdkafka/tests/0150-telemetry_mock.c +651 -0
  437. package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
  438. package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
  439. package/deps/librdkafka/tests/0153-memberid.c +128 -0
  440. package/deps/librdkafka/tests/0155-share_group_heartbeat_mock.c +2695 -0
  441. package/deps/librdkafka/tests/0156-share_consumer_fetch_mock.c +1551 -0
  442. package/deps/librdkafka/tests/0157-share_consumer_ack_mock.c +1788 -0
  443. package/deps/librdkafka/tests/0158-share_consumer_transactions_mock.c +467 -0
  444. package/deps/librdkafka/tests/0170-share_consumer_subscription.c +2192 -0
  445. package/deps/librdkafka/tests/0171-share_consumer_consume.c +2150 -0
  446. package/deps/librdkafka/tests/0172-share_consumer_acknowledge.c +1919 -0
  447. package/deps/librdkafka/tests/0173-share_consumer_commit_async.c +2735 -0
  448. package/deps/librdkafka/tests/0174-share_consumer_concurrency.c +1434 -0
  449. package/deps/librdkafka/tests/0175-share_consumer_groups.c +574 -0
  450. package/deps/librdkafka/tests/0176-share_consumer_commit_sync.c +2986 -0
  451. package/deps/librdkafka/tests/0177-share_consumer_transactions.c +1815 -0
  452. package/deps/librdkafka/tests/0178-share_consumer_close.c +2102 -0
  453. package/deps/librdkafka/tests/0179-share_consumer_destroy.c +2482 -0
  454. package/deps/librdkafka/tests/0180-share_consumer_config.c +1677 -0
  455. package/deps/librdkafka/tests/0181-share_consumer_topic_delete.c +354 -0
  456. package/deps/librdkafka/tests/0182-share_consumer_error_handling_mock.c +3358 -0
  457. package/deps/librdkafka/tests/0183-share_consumer_leader_change_mock.c +1234 -0
  458. package/deps/librdkafka/tests/0184-share_consumer_topic_recreate.c +1356 -0
  459. package/deps/librdkafka/tests/0185-share_consumer_max_poll_interval.c +557 -0
  460. package/deps/librdkafka/tests/0186-share_consumer_fatal_error.c +360 -0
  461. package/deps/librdkafka/tests/0190-share_consumer_telemetry.c +387 -0
  462. package/deps/librdkafka/tests/1000-unktopic.c +164 -0
  463. package/deps/librdkafka/tests/8000-idle.cpp +60 -0
  464. package/deps/librdkafka/tests/8001-fetch_from_follower_mock_manual.c +113 -0
  465. package/deps/librdkafka/tests/CMakeLists.txt +193 -0
  466. package/deps/librdkafka/tests/LibrdkafkaTestApp.py +291 -0
  467. package/deps/librdkafka/tests/Makefile +182 -0
  468. package/deps/librdkafka/tests/README.md +514 -0
  469. package/deps/librdkafka/tests/autotest.sh +33 -0
  470. package/deps/librdkafka/tests/backtrace.gdb +30 -0
  471. package/deps/librdkafka/tests/broker_version_tests.py +315 -0
  472. package/deps/librdkafka/tests/buildbox.sh +17 -0
  473. package/deps/librdkafka/tests/cleanup-checker-tests.sh +20 -0
  474. package/deps/librdkafka/tests/cluster_testing.py +203 -0
  475. package/deps/librdkafka/tests/delete-test-topics.sh +56 -0
  476. package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
  477. package/deps/librdkafka/tests/fixtures/ssl/Makefile +8 -0
  478. package/deps/librdkafka/tests/fixtures/ssl/README.md +13 -0
  479. package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
  480. package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
  481. package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
  482. package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +50 -0
  483. package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
  484. package/deps/librdkafka/tests/fixtures/ssl/client2.key +46 -0
  485. package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +168 -0
  486. package/deps/librdkafka/tests/fuzzers/Makefile +12 -0
  487. package/deps/librdkafka/tests/fuzzers/README.md +31 -0
  488. package/deps/librdkafka/tests/fuzzers/fuzz_regex.c +74 -0
  489. package/deps/librdkafka/tests/fuzzers/helpers.h +90 -0
  490. package/deps/librdkafka/tests/gen-ssl-certs.sh +165 -0
  491. package/deps/librdkafka/tests/interactive_broker_version.py +170 -0
  492. package/deps/librdkafka/tests/interceptor_test/CMakeLists.txt +16 -0
  493. package/deps/librdkafka/tests/interceptor_test/Makefile +22 -0
  494. package/deps/librdkafka/tests/interceptor_test/interceptor_test.c +314 -0
  495. package/deps/librdkafka/tests/interceptor_test/interceptor_test.h +54 -0
  496. package/deps/librdkafka/tests/java/IncrementalRebalanceCli.java +97 -0
  497. package/deps/librdkafka/tests/java/Makefile +13 -0
  498. package/deps/librdkafka/tests/java/Murmur2Cli.java +46 -0
  499. package/deps/librdkafka/tests/java/README.md +14 -0
  500. package/deps/librdkafka/tests/java/TransactionProducerCli.java +162 -0
  501. package/deps/librdkafka/tests/java/run-class.sh +11 -0
  502. package/deps/librdkafka/tests/librdkafka.suppressions +483 -0
  503. package/deps/librdkafka/tests/lz4_manual_test.sh +59 -0
  504. package/deps/librdkafka/tests/multi-broker-version-test.sh +50 -0
  505. package/deps/librdkafka/tests/parse-refcnt.sh +43 -0
  506. package/deps/librdkafka/tests/performance_plot.py +115 -0
  507. package/deps/librdkafka/tests/plugin_test/Makefile +19 -0
  508. package/deps/librdkafka/tests/plugin_test/plugin_test.c +58 -0
  509. package/deps/librdkafka/tests/requirements.txt +2 -0
  510. package/deps/librdkafka/tests/run-all-tests.sh +81 -0
  511. package/deps/librdkafka/tests/run-consumer-tests.sh +16 -0
  512. package/deps/librdkafka/tests/run-producer-tests.sh +16 -0
  513. package/deps/librdkafka/tests/run-test-batches.py +157 -0
  514. package/deps/librdkafka/tests/run-test.sh +141 -0
  515. package/deps/librdkafka/tests/rusage.c +249 -0
  516. package/deps/librdkafka/tests/sasl_test.py +289 -0
  517. package/deps/librdkafka/tests/scenarios/README.md +6 -0
  518. package/deps/librdkafka/tests/scenarios/ak23.json +6 -0
  519. package/deps/librdkafka/tests/scenarios/default.json +5 -0
  520. package/deps/librdkafka/tests/scenarios/noautocreate.json +5 -0
  521. package/deps/librdkafka/tests/share_telemetry_verify/share-otel-collector-config.yaml +28 -0
  522. package/deps/librdkafka/tests/sockem.c +801 -0
  523. package/deps/librdkafka/tests/sockem.h +85 -0
  524. package/deps/librdkafka/tests/sockem_ctrl.c +145 -0
  525. package/deps/librdkafka/tests/sockem_ctrl.h +61 -0
  526. package/deps/librdkafka/tests/test.c +8376 -0
  527. package/deps/librdkafka/tests/test.conf.example +27 -0
  528. package/deps/librdkafka/tests/test.h +1150 -0
  529. package/deps/librdkafka/tests/testcpp.cpp +131 -0
  530. package/deps/librdkafka/tests/testcpp.h +388 -0
  531. package/deps/librdkafka/tests/testshared.h +416 -0
  532. package/deps/librdkafka/tests/tools/README.md +4 -0
  533. package/deps/librdkafka/tests/tools/stats/README.md +21 -0
  534. package/deps/librdkafka/tests/tools/stats/filter.jq +42 -0
  535. package/deps/librdkafka/tests/tools/stats/graph.py +150 -0
  536. package/deps/librdkafka/tests/tools/stats/requirements.txt +3 -0
  537. package/deps/librdkafka/tests/tools/stats/to_csv.py +124 -0
  538. package/deps/librdkafka/tests/trivup/trivup-0.15.0.tar.gz +0 -0
  539. package/deps/librdkafka/tests/until-fail.sh +87 -0
  540. package/deps/librdkafka/tests/xxxx-assign_partition.c +122 -0
  541. package/deps/librdkafka/tests/xxxx-metadata.cpp +159 -0
  542. package/deps/librdkafka/vcpkg.json +23 -0
  543. package/deps/librdkafka/win32/README.md +5 -0
  544. package/deps/librdkafka/win32/build-package.bat +3 -0
  545. package/deps/librdkafka/win32/build.bat +19 -0
  546. package/deps/librdkafka/win32/common.vcxproj +84 -0
  547. package/deps/librdkafka/win32/interceptor_test/interceptor_test.vcxproj +87 -0
  548. package/deps/librdkafka/win32/librdkafka.autopkg.template +54 -0
  549. package/deps/librdkafka/win32/librdkafka.master.testing.targets +13 -0
  550. package/deps/librdkafka/win32/librdkafka.sln +226 -0
  551. package/deps/librdkafka/win32/librdkafka.vcxproj +282 -0
  552. package/deps/librdkafka/win32/librdkafkacpp/librdkafkacpp.vcxproj +104 -0
  553. package/deps/librdkafka/win32/msbuild.ps1 +58 -0
  554. package/deps/librdkafka/win32/openssl_engine_example/openssl_engine_example.vcxproj +132 -0
  555. package/deps/librdkafka/win32/package-zip.ps1 +46 -0
  556. package/deps/librdkafka/win32/packages/repositories.config +4 -0
  557. package/deps/librdkafka/win32/push-package.bat +4 -0
  558. package/deps/librdkafka/win32/rdkafka_complex_consumer_example_cpp/rdkafka_complex_consumer_example_cpp.vcxproj +67 -0
  559. package/deps/librdkafka/win32/rdkafka_example/rdkafka_example.vcxproj +97 -0
  560. package/deps/librdkafka/win32/rdkafka_performance/rdkafka_performance.vcxproj +97 -0
  561. package/deps/librdkafka/win32/setup-msys2.ps1 +47 -0
  562. package/deps/librdkafka/win32/setup-vcpkg.ps1 +34 -0
  563. package/deps/librdkafka/win32/tests/test.conf.example +25 -0
  564. package/deps/librdkafka/win32/tests/tests.vcxproj +278 -0
  565. package/deps/librdkafka/win32/win_ssl_cert_store/win_ssl_cert_store.vcxproj +132 -0
  566. package/deps/librdkafka/win32/wingetopt.c +564 -0
  567. package/deps/librdkafka/win32/wingetopt.h +101 -0
  568. package/deps/librdkafka/win32/wintime.h +33 -0
  569. package/deps/librdkafka.gyp +83 -0
  570. package/deps/windows-install.py +80 -0
  571. package/index.d.ts +3 -0
  572. package/lib/admin.js +732 -0
  573. package/lib/client.js +679 -0
  574. package/lib/error.js +501 -0
  575. package/lib/index.js +8 -0
  576. package/lib/kafka-consumer-stream.js +399 -0
  577. package/lib/kafka-consumer.js +798 -0
  578. package/lib/kafkajs/_admin.js +1052 -0
  579. package/lib/kafkajs/_common.js +887 -0
  580. package/lib/kafkajs/_consumer.js +2156 -0
  581. package/lib/kafkajs/_consumer_cache.js +310 -0
  582. package/lib/kafkajs/_error.js +298 -0
  583. package/lib/kafkajs/_kafka.js +125 -0
  584. package/lib/kafkajs/_linked-list.js +220 -0
  585. package/lib/kafkajs/_producer.js +863 -0
  586. package/lib/kafkajs/index.js +5 -0
  587. package/lib/producer/high-level-producer.js +400 -0
  588. package/lib/producer-stream.js +337 -0
  589. package/lib/producer.js +405 -0
  590. package/lib/rdkafka.js +44 -0
  591. package/lib/tools/ref-counter.js +53 -0
  592. package/lib/topic-partition.js +91 -0
  593. package/lib/topic.js +43 -0
  594. package/lib/util.js +55 -0
  595. package/librdkafka.js +12 -0
  596. package/package.json +85 -0
  597. package/shim/source_location +54 -0
  598. package/src/admin.cc +1597 -0
  599. package/src/admin.h +112 -0
  600. package/src/binding.cc +76 -0
  601. package/src/binding.h +25 -0
  602. package/src/callbacks.cc +676 -0
  603. package/src/callbacks.h +302 -0
  604. package/src/common.cc +1628 -0
  605. package/src/common.h +192 -0
  606. package/src/config.cc +219 -0
  607. package/src/config.h +61 -0
  608. package/src/connection.cc +703 -0
  609. package/src/connection.h +117 -0
  610. package/src/errors.cc +119 -0
  611. package/src/errors.h +60 -0
  612. package/src/kafka-consumer.cc +1539 -0
  613. package/src/kafka-consumer.h +147 -0
  614. package/src/producer.cc +917 -0
  615. package/src/producer.h +124 -0
  616. package/src/topic.cc +173 -0
  617. package/src/topic.h +54 -0
  618. package/src/workers.cc +1714 -0
  619. package/src/workers.h +690 -0
  620. package/types/config.d.ts +1269 -0
  621. package/types/errors.d.ts +373 -0
  622. package/types/kafkajs.d.ts +568 -0
  623. package/types/rdkafka.d.ts +571 -0
  624. package/util/configure.js +32 -0
  625. package/util/generate-docs.sh +18 -0
  626. package/util/get-env.js +6 -0
  627. package/util/test-compile.js +11 -0
  628. package/util/test-producer-delivery.js +100 -0
@@ -0,0 +1,3820 @@
1
+ /*
2
+ * librdkafka - The Apache Kafka C/C++ library
3
+ *
4
+ * Copyright (c) 2022, Magnus Edenhill
5
+ * 2023, Confluent Inc.
6
+ * All rights reserved.
7
+ *
8
+ * Redistribution and use in source and binary forms, with or without
9
+ * modification, are permitted provided that the following conditions are met:
10
+ *
11
+ * 1. Redistributions of source code must retain the above copyright notice,
12
+ * this list of conditions and the following disclaimer.
13
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
14
+ * this list of conditions and the following disclaimer in the documentation
15
+ * and/or other materials provided with the distribution.
16
+ *
17
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
+ * POSSIBILITY OF SUCH DAMAGE.
28
+ */
29
+
30
+
31
+ /**
32
+ * @name Fetcher
33
+ *
34
+ */
35
+
36
+ #include "rdkafka_int.h"
37
+ #include "rdkafka_offset.h"
38
+ #include "rdkafka_msgset.h"
39
+ #include "rdkafka_fetcher.h"
40
+ #include "rdkafka_request.h"
41
+ #include "rdkafka_share_acknowledgement.h"
42
+
43
+
44
+ /**
45
+ * Backoff the next Fetch request (due to error).
46
+ */
47
+ static void rd_kafka_broker_fetch_backoff(rd_kafka_broker_t *rkb,
48
+ rd_kafka_resp_err_t err) {
49
+ int backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms;
50
+ rkb->rkb_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000);
51
+ rd_rkb_dbg(rkb, FETCH, "BACKOFF", "Fetch backoff for %dms: %s",
52
+ backoff_ms, rd_kafka_err2str(err));
53
+ }
54
+
55
+ /**
56
+ * @brief Backoff the next Fetch for specific partition
57
+ *
58
+ * @returns the absolute backoff time (the current time for no backoff).
59
+ */
60
+ static rd_ts_t rd_kafka_toppar_fetch_backoff(rd_kafka_broker_t *rkb,
61
+ rd_kafka_toppar_t *rktp,
62
+ rd_kafka_resp_err_t err) {
63
+ int backoff_ms;
64
+
65
+ /* Don't back off on reaching end of partition */
66
+ if (err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
67
+ rktp->rktp_ts_fetch_backoff = 0;
68
+ return rd_clock(); /* Immediate: No practical backoff */
69
+ }
70
+
71
+ if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL)
72
+ backoff_ms = rkb->rkb_rk->rk_conf.fetch_queue_backoff_ms;
73
+ else
74
+ backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms;
75
+
76
+ if (unlikely(!backoff_ms)) {
77
+ rktp->rktp_ts_fetch_backoff = 0;
78
+ return rd_clock(); /* Immediate: No practical backoff */
79
+ }
80
+
81
+ /* Certain errors that may require manual intervention should have
82
+ * a longer backoff time. */
83
+ if (err == RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED)
84
+ backoff_ms = RD_MAX(1000, backoff_ms * 10);
85
+
86
+ rktp->rktp_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000);
87
+
88
+ rd_rkb_dbg(rkb, FETCH, "BACKOFF",
89
+ "%s [%" PRId32 "]: Fetch backoff for %dms%s%s",
90
+ rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
91
+ backoff_ms, err ? ": " : "",
92
+ err ? rd_kafka_err2str(err) : "");
93
+
94
+ return rktp->rktp_ts_fetch_backoff;
95
+ }
96
+
97
+ /**
98
+ * @brief Handle preferred replica in fetch response.
99
+ *
100
+ * @locks rd_kafka_toppar_lock(rktp) and
101
+ * rd_kafka_rdlock(rk) must NOT be held.
102
+ *
103
+ * @locality broker thread
104
+ */
105
+ static void rd_kafka_fetch_preferred_replica_handle(rd_kafka_toppar_t *rktp,
106
+ rd_kafka_buf_t *rkbuf,
107
+ rd_kafka_broker_t *rkb,
108
+ int32_t preferred_id) {
109
+ const rd_ts_t one_minute = 60 * 1000 * 1000;
110
+ const rd_ts_t five_seconds = 5 * 1000 * 1000;
111
+ rd_kafka_broker_t *preferred_rkb;
112
+ rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
113
+ rd_ts_t new_intvl =
114
+ rd_interval_immediate(&rktp->rktp_new_lease_intvl, one_minute, 0);
115
+
116
+ if (new_intvl < 0) {
117
+ /* In lieu of KIP-320, the toppar is delegated back to
118
+ * the leader in the event of an offset out-of-range
119
+ * error (KIP-392 error case #4) because this scenario
120
+ * implies the preferred replica is out-of-sync.
121
+ *
122
+ * If program execution reaches here, the leader has
123
+ * relatively quickly instructed the client back to
124
+ * a preferred replica, quite possibly the same one
125
+ * as before (possibly resulting from stale metadata),
126
+ * so we back off the toppar to slow down potential
127
+ * back-and-forth.
128
+ */
129
+
130
+ if (rd_interval_immediate(&rktp->rktp_new_lease_log_intvl,
131
+ one_minute, 0) > 0)
132
+ rd_rkb_log(rkb, LOG_NOTICE, "FETCH",
133
+ "%.*s [%" PRId32
134
+ "]: preferred replica "
135
+ "(%" PRId32
136
+ ") lease changing too quickly "
137
+ "(%" PRId64
138
+ "s < 60s): possibly due to "
139
+ "unavailable replica or stale cluster "
140
+ "state: backing off next fetch",
141
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
142
+ rktp->rktp_partition, preferred_id,
143
+ (one_minute - -new_intvl) / (1000 * 1000));
144
+
145
+ rd_kafka_toppar_fetch_backoff(rkb, rktp,
146
+ RD_KAFKA_RESP_ERR_NO_ERROR);
147
+ }
148
+
149
+ rd_kafka_rdlock(rk);
150
+ preferred_rkb = rd_kafka_broker_find_by_nodeid(rk, preferred_id);
151
+ rd_kafka_rdunlock(rk);
152
+
153
+ if (preferred_rkb) {
154
+ rd_interval_reset_to_now(&rktp->rktp_lease_intvl, 0);
155
+ rd_kafka_toppar_lock(rktp);
156
+ rd_kafka_toppar_broker_update(rktp, preferred_id, preferred_rkb,
157
+ "preferred replica updated");
158
+ rd_kafka_toppar_unlock(rktp);
159
+ rd_kafka_broker_destroy(preferred_rkb);
160
+ return;
161
+ }
162
+
163
+ if (rd_interval_immediate(&rktp->rktp_metadata_intvl, five_seconds, 0) >
164
+ 0) {
165
+ rd_rkb_log(rkb, LOG_NOTICE, "FETCH",
166
+ "%.*s [%" PRId32 "]: preferred replica (%" PRId32
167
+ ") "
168
+ "is unknown: refreshing metadata",
169
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
170
+ rktp->rktp_partition, preferred_id);
171
+
172
+ rd_kafka_metadata_refresh_brokers(
173
+ rktp->rktp_rkt->rkt_rk, NULL,
174
+ "preferred replica unavailable");
175
+ }
176
+
177
+ rd_kafka_toppar_fetch_backoff(rkb, rktp,
178
+ RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE);
179
+ }
180
+
181
+
182
+ /**
183
+ * @brief Handle partition-specific Fetch error.
184
+ */
185
+ static void rd_kafka_fetch_reply_handle_partition_error(
186
+ rd_kafka_broker_t *rkb,
187
+ rd_kafka_toppar_t *rktp,
188
+ const struct rd_kafka_toppar_ver *tver,
189
+ rd_kafka_resp_err_t err,
190
+ int64_t HighwaterMarkOffset) {
191
+
192
+ rd_rkb_dbg(rkb, FETCH, "FETCHERR",
193
+ "%.*s [%" PRId32 "]: Fetch failed at %s: %s",
194
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
195
+ rktp->rktp_partition,
196
+ rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos),
197
+ rd_kafka_err2name(err));
198
+
199
+ /* Some errors should be passed to the
200
+ * application while some handled by rdkafka */
201
+ switch (err) {
202
+ /* Errors handled by rdkafka */
203
+ case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE:
204
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
205
+ case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE:
206
+ case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER:
207
+ case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE:
208
+ case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE:
209
+ case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR:
210
+ case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH:
211
+ case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH:
212
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
213
+ if (err == RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE) {
214
+ /* Occurs when:
215
+ * - Msg exists on broker but
216
+ * offset > HWM, or:
217
+ * - HWM is >= offset, but msg not
218
+ * yet available at that offset
219
+ * (replica is out of sync).
220
+ * - partition leader is out of sync.
221
+ *
222
+ * Handle by requesting metadata update, changing back
223
+ * to the leader, and then retrying FETCH
224
+ * (with backoff).
225
+ */
226
+ rd_rkb_dbg(rkb, MSG, "FETCH",
227
+ "Topic %s [%" PRId32
228
+ "]: %s not "
229
+ "available on broker %" PRId32
230
+ " (leader %" PRId32
231
+ "): updating metadata and retrying",
232
+ rktp->rktp_rkt->rkt_topic->str,
233
+ rktp->rktp_partition,
234
+ rd_kafka_fetch_pos2str(
235
+ rktp->rktp_offsets.fetch_pos),
236
+ rktp->rktp_broker_id, rktp->rktp_leader_id);
237
+ }
238
+
239
+ if (err == RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH) {
240
+ rd_rkb_dbg(rkb, MSG | RD_KAFKA_DBG_CONSUMER, "FETCH",
241
+ "Topic %s [%" PRId32
242
+ "]: Fetch failed at %s: %s: broker %" PRId32
243
+ "has not yet caught up on latest metadata: "
244
+ "retrying",
245
+ rktp->rktp_rkt->rkt_topic->str,
246
+ rktp->rktp_partition,
247
+ rd_kafka_fetch_pos2str(
248
+ rktp->rktp_offsets.fetch_pos),
249
+ rd_kafka_err2str(err), rktp->rktp_broker_id);
250
+ }
251
+
252
+ if (rktp->rktp_broker_id != rktp->rktp_leader_id) {
253
+ rd_kafka_toppar_delegate_to_leader(rktp);
254
+ }
255
+ /* Request metadata information update*/
256
+ rd_kafka_toppar_leader_unavailable(rktp, "fetch", err);
257
+ break;
258
+
259
+ case RD_KAFKA_RESP_ERR_OFFSET_OUT_OF_RANGE: {
260
+ rd_kafka_fetch_pos_t err_pos;
261
+
262
+ if (rktp->rktp_broker_id != rktp->rktp_leader_id &&
263
+ rktp->rktp_offsets.fetch_pos.offset > HighwaterMarkOffset) {
264
+ rd_kafka_log(rkb->rkb_rk, LOG_WARNING, "FETCH",
265
+ "Topic %s [%" PRId32
266
+ "]: %s "
267
+ " out of range (HighwaterMark %" PRId64
268
+ " fetching from "
269
+ "broker %" PRId32 " (leader %" PRId32
270
+ "): reverting to leader",
271
+ rktp->rktp_rkt->rkt_topic->str,
272
+ rktp->rktp_partition,
273
+ rd_kafka_fetch_pos2str(
274
+ rktp->rktp_offsets.fetch_pos),
275
+ HighwaterMarkOffset, rktp->rktp_broker_id,
276
+ rktp->rktp_leader_id);
277
+
278
+ /* Out of range error cannot be taken as definitive
279
+ * when fetching from follower.
280
+ * Revert back to the leader in lieu of KIP-320.
281
+ */
282
+ rd_kafka_toppar_delegate_to_leader(rktp);
283
+ break;
284
+ }
285
+
286
+ /* Application error */
287
+ err_pos = rktp->rktp_offsets.fetch_pos;
288
+ rktp->rktp_offsets.fetch_pos.offset = RD_KAFKA_OFFSET_INVALID;
289
+ rktp->rktp_offsets.fetch_pos.leader_epoch = -1;
290
+ rd_kafka_offset_reset(rktp, rd_kafka_broker_id(rkb), err_pos,
291
+ err,
292
+ "fetch failed due to requested offset "
293
+ "not available on the broker");
294
+ } break;
295
+
296
+ case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED:
297
+ /* If we're not authorized to access the
298
+ * topic mark it as errored to deny
299
+ * further Fetch requests. */
300
+ if (rktp->rktp_last_error != err) {
301
+ rd_kafka_consumer_err(
302
+ rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err,
303
+ tver->version, NULL, rktp,
304
+ rktp->rktp_offsets.fetch_pos.offset,
305
+ "Fetch from broker %" PRId32 " failed: %s",
306
+ rd_kafka_broker_id(rkb), rd_kafka_err2str(err));
307
+ rktp->rktp_last_error = err;
308
+ }
309
+ break;
310
+
311
+
312
+ /* Application errors */
313
+ case RD_KAFKA_RESP_ERR__PARTITION_EOF:
314
+ if (rkb->rkb_rk->rk_conf.enable_partition_eof)
315
+ rd_kafka_consumer_err(
316
+ rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err,
317
+ tver->version, NULL, rktp,
318
+ rktp->rktp_offsets.fetch_pos.offset,
319
+ "Fetch from broker %" PRId32
320
+ " reached end of "
321
+ "partition at offset %" PRId64
322
+ " (HighwaterMark %" PRId64 ")",
323
+ rd_kafka_broker_id(rkb),
324
+ rktp->rktp_offsets.fetch_pos.offset,
325
+ HighwaterMarkOffset);
326
+ break;
327
+
328
+ case RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE:
329
+ default: /* and all other errors */
330
+ rd_dassert(tver->version > 0);
331
+ rd_kafka_consumer_err(
332
+ rktp->rktp_fetchq, rd_kafka_broker_id(rkb), err,
333
+ tver->version, NULL, rktp,
334
+ rktp->rktp_offsets.fetch_pos.offset,
335
+ "Fetch from broker %" PRId32 " failed at %s: %s",
336
+ rd_kafka_broker_id(rkb),
337
+ rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos),
338
+ rd_kafka_err2str(err));
339
+ break;
340
+ }
341
+
342
+ /* Back off the next fetch for this partition */
343
+ rd_kafka_toppar_fetch_backoff(rkb, rktp, err);
344
+ }
345
+
346
+ static void rd_kafkap_Fetch_reply_tags_set_topic_cnt(
347
+ rd_kafkap_Fetch_reply_tags_t *reply_tags,
348
+ int32_t TopicCnt) {
349
+ reply_tags->TopicCnt = TopicCnt;
350
+ rd_dassert(!reply_tags->Topics);
351
+ reply_tags->Topics = rd_calloc(TopicCnt, sizeof(*reply_tags->Topics));
352
+ }
353
+
354
+ static void
355
+ rd_kafkap_Fetch_reply_tags_set_topic(rd_kafkap_Fetch_reply_tags_t *reply_tags,
356
+ int TopicIdx,
357
+ rd_kafka_Uuid_t TopicId,
358
+ int32_t PartitionCnt) {
359
+ reply_tags->Topics[TopicIdx].TopicId = TopicId;
360
+ reply_tags->Topics[TopicIdx].PartitionCnt = PartitionCnt;
361
+ rd_dassert(!reply_tags->Topics[TopicIdx].Partitions);
362
+ reply_tags->Topics[TopicIdx].Partitions = rd_calloc(
363
+ PartitionCnt, sizeof(*reply_tags->Topics[TopicIdx].Partitions));
364
+ }
365
+
366
+
367
+ static void
368
+ rd_kafkap_Fetch_reply_tags_destroy(rd_kafkap_Fetch_reply_tags_t *reply_tags) {
369
+ int i;
370
+ for (i = 0; i < reply_tags->TopicCnt; i++) {
371
+ RD_IF_FREE(reply_tags->Topics[i].Partitions, rd_free);
372
+ }
373
+ RD_IF_FREE(reply_tags->Topics, rd_free);
374
+ RD_IF_FREE(reply_tags->NodeEndpoints.NodeEndpoints, rd_free);
375
+ }
376
+
377
+ static int rd_kafkap_Fetch_reply_tags_partition_parse(
378
+ rd_kafka_buf_t *rkbuf,
379
+ uint64_t tagtype,
380
+ uint64_t taglen,
381
+ rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags,
382
+ rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) {
383
+ switch (tagtype) {
384
+ case 1: /* CurrentLeader */
385
+ if (rd_kafka_buf_read_CurrentLeader(
386
+ rkbuf, &PartitionTags->CurrentLeader) == -1)
387
+ goto err_parse;
388
+ TopicTags->partitions_with_leader_change_cnt++;
389
+ return 1;
390
+ default:
391
+ return 0;
392
+ }
393
+ err_parse:
394
+ return -1;
395
+ }
396
+
397
+ static int
398
+ rd_kafkap_Fetch_reply_tags_parse(rd_kafka_buf_t *rkbuf,
399
+ uint64_t tagtype,
400
+ uint64_t taglen,
401
+ rd_kafkap_Fetch_reply_tags_t *tags) {
402
+ switch (tagtype) {
403
+ case 0: /* NodeEndpoints */
404
+ if (rd_kafka_buf_read_NodeEndpoints(rkbuf,
405
+ &tags->NodeEndpoints) == -1)
406
+ goto err_parse;
407
+ return 1;
408
+ default:
409
+ return 0;
410
+ }
411
+ err_parse:
412
+ return -1;
413
+ }
414
+
415
+ static void
416
+ rd_kafka_handle_Fetch_metadata_update(rd_kafka_broker_t *rkb,
417
+ rd_kafkap_Fetch_reply_tags_t *FetchTags) {
418
+ if (FetchTags->topics_with_leader_change_cnt &&
419
+ FetchTags->NodeEndpoints.NodeEndpoints) {
420
+ rd_kafka_metadata_t *md = NULL;
421
+ rd_kafka_metadata_internal_t *mdi = NULL;
422
+ rd_tmpabuf_t tbuf;
423
+ int32_t nodeid;
424
+ rd_kafka_op_t *rko;
425
+ int i, changed_topic, changed_partition;
426
+
427
+ rd_kafka_broker_lock(rkb);
428
+ nodeid = rkb->rkb_nodeid;
429
+ rd_kafka_broker_unlock(rkb);
430
+
431
+ rd_tmpabuf_new(&tbuf, 0, rd_true /*assert on fail*/);
432
+ rd_tmpabuf_add_alloc(&tbuf, sizeof(*mdi));
433
+ rd_kafkap_leader_discovery_tmpabuf_add_alloc_brokers(
434
+ &tbuf, &FetchTags->NodeEndpoints);
435
+ rd_kafkap_leader_discovery_tmpabuf_add_alloc_topics(
436
+ &tbuf, FetchTags->topics_with_leader_change_cnt);
437
+ for (i = 0; i < FetchTags->TopicCnt; i++) {
438
+ if (!FetchTags->Topics[i]
439
+ .partitions_with_leader_change_cnt)
440
+ continue;
441
+ rd_kafkap_leader_discovery_tmpabuf_add_alloc_topic(
442
+ &tbuf, NULL,
443
+ FetchTags->Topics[i]
444
+ .partitions_with_leader_change_cnt);
445
+ }
446
+ rd_tmpabuf_finalize(&tbuf);
447
+
448
+ mdi = rd_tmpabuf_alloc(&tbuf, sizeof(*mdi));
449
+ md = &mdi->metadata;
450
+
451
+ rd_kafkap_leader_discovery_metadata_init(mdi, nodeid);
452
+
453
+ rd_kafkap_leader_discovery_set_brokers(
454
+ &tbuf, mdi, &FetchTags->NodeEndpoints);
455
+
456
+ rd_kafkap_leader_discovery_set_topic_cnt(
457
+ &tbuf, mdi, FetchTags->topics_with_leader_change_cnt);
458
+
459
+ changed_topic = 0;
460
+ for (i = 0; i < FetchTags->TopicCnt; i++) {
461
+ int j;
462
+ if (!FetchTags->Topics[i]
463
+ .partitions_with_leader_change_cnt)
464
+ continue;
465
+
466
+ rd_kafkap_leader_discovery_set_topic(
467
+ &tbuf, mdi, changed_topic,
468
+ FetchTags->Topics[i].TopicId, NULL,
469
+ FetchTags->Topics[i]
470
+ .partitions_with_leader_change_cnt);
471
+
472
+ changed_partition = 0;
473
+ for (j = 0; j < FetchTags->Topics[i].PartitionCnt;
474
+ j++) {
475
+ if (FetchTags->Topics[i]
476
+ .Partitions[j]
477
+ .CurrentLeader.LeaderId < 0)
478
+ continue;
479
+
480
+ rd_kafkap_Fetch_reply_tags_Partition_t
481
+ *Partition =
482
+ &FetchTags->Topics[i].Partitions[j];
483
+ rd_kafkap_leader_discovery_set_CurrentLeader(
484
+ &tbuf, mdi, changed_topic,
485
+ changed_partition, Partition->Partition,
486
+ &Partition->CurrentLeader);
487
+ changed_partition++;
488
+ }
489
+ changed_topic++;
490
+ }
491
+
492
+ rko = rd_kafka_op_new(RD_KAFKA_OP_METADATA_UPDATE);
493
+ rko->rko_u.metadata.md = md;
494
+ rko->rko_u.metadata.mdi = mdi;
495
+ rd_kafka_q_enq(rkb->rkb_rk->rk_ops, rko);
496
+ }
497
+ }
498
+
499
+ /**
500
+ * @brief Per-partition FetchResponse parsing and handling.
501
+ *
502
+ * @returns an error on buffer parse failure, else RD_KAFKA_RESP_ERR_NO_ERROR.
503
+ */
504
+ static rd_kafka_resp_err_t rd_kafka_fetch_reply_handle_partition(
505
+ rd_kafka_broker_t *rkb,
506
+ const rd_kafkap_str_t *topic,
507
+ rd_kafka_topic_t *rkt /*possibly NULL*/,
508
+ rd_kafka_buf_t *rkbuf,
509
+ rd_kafka_buf_t *request,
510
+ int16_t ErrorCode,
511
+ rd_kafkap_Fetch_reply_tags_Topic_t *TopicTags,
512
+ rd_kafkap_Fetch_reply_tags_Partition_t *PartitionTags) {
513
+ const int log_decode_errors = LOG_ERR;
514
+ struct rd_kafka_toppar_ver *tver, tver_skel;
515
+ rd_kafka_toppar_t *rktp = NULL;
516
+ rd_kafka_aborted_txns_t *aborted_txns = NULL;
517
+ rd_slice_t save_slice;
518
+ int32_t fetch_version;
519
+ struct {
520
+ int32_t Partition;
521
+ int16_t ErrorCode;
522
+ int64_t HighwaterMarkOffset;
523
+ int64_t LastStableOffset; /* v4 */
524
+ int64_t LogStartOffset; /* v5 */
525
+ int32_t MessageSetSize;
526
+ int32_t PreferredReadReplica; /* v11 */
527
+ } hdr;
528
+ rd_kafka_resp_err_t err;
529
+ int64_t end_offset;
530
+
531
+ rd_kafka_buf_read_i32(rkbuf, &hdr.Partition);
532
+ rd_kafka_buf_read_i16(rkbuf, &hdr.ErrorCode);
533
+ if (PartitionTags)
534
+ PartitionTags->Partition = hdr.Partition;
535
+ if (ErrorCode)
536
+ hdr.ErrorCode = ErrorCode;
537
+ rd_kafka_buf_read_i64(rkbuf, &hdr.HighwaterMarkOffset);
538
+
539
+ end_offset = hdr.HighwaterMarkOffset;
540
+
541
+ hdr.LastStableOffset = RD_KAFKA_OFFSET_INVALID;
542
+ hdr.LogStartOffset = RD_KAFKA_OFFSET_INVALID;
543
+ if (rd_kafka_buf_ApiVersion(request) >= 4) {
544
+ int32_t AbortedTxnCnt;
545
+ int k;
546
+ rd_kafka_buf_read_i64(rkbuf, &hdr.LastStableOffset);
547
+ if (rd_kafka_buf_ApiVersion(request) >= 5)
548
+ rd_kafka_buf_read_i64(rkbuf, &hdr.LogStartOffset);
549
+
550
+ rd_kafka_buf_read_arraycnt(rkbuf, &AbortedTxnCnt,
551
+ RD_KAFKAP_ABORTED_TRANSACTIONS_MAX);
552
+
553
+ if (rkb->rkb_rk->rk_conf.isolation_level ==
554
+ RD_KAFKA_READ_UNCOMMITTED) {
555
+
556
+ if (unlikely(AbortedTxnCnt > 0)) {
557
+ rd_rkb_log(rkb, LOG_ERR, "FETCH",
558
+ "%.*s [%" PRId32
559
+ "]: "
560
+ "%" PRId32
561
+ " aborted transaction(s) "
562
+ "encountered in READ_UNCOMMITTED "
563
+ "fetch response: ignoring.",
564
+ RD_KAFKAP_STR_PR(topic),
565
+ hdr.Partition, AbortedTxnCnt);
566
+ for (k = 0; k < AbortedTxnCnt; k++) {
567
+ rd_kafka_buf_skip(rkbuf, (8 + 8));
568
+ /* AbortedTransaction tags */
569
+ rd_kafka_buf_skip_tags(rkbuf);
570
+ }
571
+ }
572
+ } else {
573
+ /* Older brokers may return LSO -1,
574
+ * in which case we use the HWM. */
575
+ if (hdr.LastStableOffset >= 0)
576
+ end_offset = hdr.LastStableOffset;
577
+
578
+ if (AbortedTxnCnt > 0) {
579
+ aborted_txns =
580
+ rd_kafka_aborted_txns_new(AbortedTxnCnt);
581
+ for (k = 0; k < AbortedTxnCnt; k++) {
582
+ int64_t PID;
583
+ int64_t FirstOffset;
584
+ rd_kafka_buf_read_i64(rkbuf, &PID);
585
+ rd_kafka_buf_read_i64(rkbuf,
586
+ &FirstOffset);
587
+ /* AbortedTransaction tags */
588
+ rd_kafka_buf_skip_tags(rkbuf);
589
+ rd_kafka_aborted_txns_add(
590
+ aborted_txns, PID, FirstOffset);
591
+ }
592
+ rd_kafka_aborted_txns_sort(aborted_txns);
593
+ }
594
+ }
595
+ }
596
+
597
+ if (rd_kafka_buf_ApiVersion(request) >= 11)
598
+ rd_kafka_buf_read_i32(rkbuf, &hdr.PreferredReadReplica);
599
+ else
600
+ hdr.PreferredReadReplica = -1;
601
+ /* Compact Records Array */
602
+ rd_kafka_buf_read_arraycnt(rkbuf, &hdr.MessageSetSize, -1);
603
+
604
+ if (unlikely(hdr.MessageSetSize < 0))
605
+ rd_kafka_buf_parse_fail(
606
+ rkbuf,
607
+ "%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32,
608
+ RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize);
609
+
610
+ /* Look up topic+partition */
611
+ if (likely(rkt != NULL)) {
612
+ rd_kafka_topic_rdlock(rkt);
613
+ rktp = rd_kafka_toppar_get(rkt, hdr.Partition,
614
+ 0 /*no ua-on-miss*/);
615
+ rd_kafka_topic_rdunlock(rkt);
616
+ }
617
+
618
+ if (unlikely(!rkt || !rktp)) {
619
+ rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC",
620
+ "Received Fetch response (error %hu) for unknown "
621
+ "topic %.*s [%" PRId32 "]: ignoring",
622
+ hdr.ErrorCode, RD_KAFKAP_STR_PR(topic),
623
+ hdr.Partition);
624
+ rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
625
+ goto done;
626
+ }
627
+
628
+ rd_kafka_toppar_lock(rktp);
629
+ rktp->rktp_lo_offset = hdr.LogStartOffset;
630
+ rktp->rktp_hi_offset = hdr.HighwaterMarkOffset;
631
+ /* Let the LastStable offset be the effective
632
+ * end_offset based on protocol version, that is:
633
+ * if connected to a broker that does not support
634
+ * LastStableOffset we use the HighwaterMarkOffset. */
635
+ rktp->rktp_ls_offset = end_offset;
636
+ rd_kafka_toppar_unlock(rktp);
637
+
638
+ if (hdr.PreferredReadReplica != -1) {
639
+
640
+ rd_kafka_fetch_preferred_replica_handle(
641
+ rktp, rkbuf, rkb, hdr.PreferredReadReplica);
642
+
643
+ if (unlikely(hdr.MessageSetSize != 0)) {
644
+ rd_rkb_log(rkb, LOG_WARNING, "FETCH",
645
+ "%.*s [%" PRId32
646
+ "]: Fetch response has both preferred read "
647
+ "replica and non-zero message set size: "
648
+ "%" PRId32 ": skipping messages",
649
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
650
+ rktp->rktp_partition, hdr.MessageSetSize);
651
+ rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
652
+ }
653
+ goto done;
654
+ }
655
+
656
+ rd_kafka_toppar_lock(rktp);
657
+
658
+ /* Make sure toppar hasn't moved to another broker
659
+ * during the lifetime of the request. */
660
+ if (unlikely(rktp->rktp_broker != rkb)) {
661
+ rd_kafka_toppar_unlock(rktp);
662
+ rd_rkb_dbg(rkb, MSG, "FETCH",
663
+ "%.*s [%" PRId32
664
+ "]: partition broker has changed: "
665
+ "discarding fetch response",
666
+ RD_KAFKAP_STR_PR(topic), hdr.Partition);
667
+ rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
668
+ goto done;
669
+ }
670
+
671
+ fetch_version = rktp->rktp_fetch_version;
672
+ rd_kafka_toppar_unlock(rktp);
673
+
674
+ /* Check if this Fetch is for an outdated fetch version,
675
+ * or the original rktp was removed and a new one
676
+ * created (due to partition count decreasing and
677
+ * then increasing again, which can happen in
678
+ * desynchronized clusters): if so ignore it. */
679
+ tver_skel.rktp = rktp;
680
+ tver = rd_list_find(request->rkbuf_rktp_vers, &tver_skel,
681
+ rd_kafka_toppar_ver_cmp);
682
+ rd_kafka_assert(NULL, tver);
683
+ if (tver->rktp != rktp || tver->version < fetch_version) {
684
+ rd_rkb_dbg(rkb, MSG, "DROP",
685
+ "%s [%" PRId32
686
+ "]: dropping outdated fetch response "
687
+ "(v%d < %d or old rktp)",
688
+ rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
689
+ tver->version, fetch_version);
690
+ rd_atomic64_add(&rktp->rktp_c.rx_ver_drops, 1);
691
+ rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
692
+ goto done;
693
+ }
694
+
695
+ rd_rkb_dbg(rkb, MSG, "FETCH",
696
+ "Topic %.*s [%" PRId32 "] MessageSet size %" PRId32
697
+ ", error \"%s\", MaxOffset %" PRId64 ", LSO %" PRId64
698
+ ", Ver %" PRId32 "/%" PRId32,
699
+ RD_KAFKAP_STR_PR(topic), hdr.Partition, hdr.MessageSetSize,
700
+ rd_kafka_err2str(hdr.ErrorCode), hdr.HighwaterMarkOffset,
701
+ hdr.LastStableOffset, tver->version, fetch_version);
702
+
703
+ /* If this is the last message of the queue,
704
+ * signal EOF back to the application. */
705
+ if (end_offset == rktp->rktp_offsets.fetch_pos.offset &&
706
+ rktp->rktp_offsets.eof_offset != end_offset) {
707
+ hdr.ErrorCode = RD_KAFKA_RESP_ERR__PARTITION_EOF;
708
+ rktp->rktp_offsets.eof_offset = end_offset;
709
+ }
710
+
711
+ if (unlikely(hdr.ErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR)) {
712
+ /* Handle partition-level errors. */
713
+ rd_kafka_fetch_reply_handle_partition_error(
714
+ rkb, rktp, tver, hdr.ErrorCode, hdr.HighwaterMarkOffset);
715
+
716
+ rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
717
+ goto done;
718
+ }
719
+
720
+ /* No error, clear any previous fetch error. */
721
+ rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR;
722
+
723
+ if (unlikely(hdr.MessageSetSize <= 0))
724
+ goto done;
725
+
726
+ /**
727
+ * Parse MessageSet
728
+ */
729
+ if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice,
730
+ (size_t)hdr.MessageSetSize))
731
+ rd_kafka_buf_check_len(rkbuf, hdr.MessageSetSize);
732
+
733
+ /* Parse messages */
734
+ err = rd_kafka_msgset_parse(rkbuf, request, rktp, aborted_txns, tver);
735
+
736
+
737
+ rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice);
738
+ /* Continue with next partition regardless of
739
+ * parse errors (which are partition-specific) */
740
+
741
+ /* On error: back off the fetcher for this partition */
742
+ if (unlikely(err))
743
+ rd_kafka_toppar_fetch_backoff(rkb, rktp, err);
744
+
745
+ goto done;
746
+
747
+ err_parse:
748
+ if (aborted_txns)
749
+ rd_kafka_aborted_txns_destroy(aborted_txns);
750
+ if (rktp)
751
+ rd_kafka_toppar_destroy(rktp); /*from get()*/
752
+ return rkbuf->rkbuf_err;
753
+
754
+ done:
755
+ if (aborted_txns)
756
+ rd_kafka_aborted_txns_destroy(aborted_txns);
757
+ if (likely(rktp != NULL))
758
+ rd_kafka_toppar_destroy(rktp); /*from get()*/
759
+
760
+ if (PartitionTags) {
761
+ /* Set default LeaderId and LeaderEpoch */
762
+ PartitionTags->CurrentLeader.LeaderId = -1;
763
+ PartitionTags->CurrentLeader.LeaderEpoch = -1;
764
+ }
765
+ rd_kafka_buf_read_tags(rkbuf,
766
+ rd_kafkap_Fetch_reply_tags_partition_parse,
767
+ TopicTags, PartitionTags);
768
+
769
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
770
+ }
771
+
772
+ /**
773
+ * Parses and handles a Fetch reply.
774
+ * Returns 0 on success or an error code on failure.
775
+ */
776
+ static rd_kafka_resp_err_t
777
+ rd_kafka_fetch_reply_handle(rd_kafka_broker_t *rkb,
778
+ rd_kafka_buf_t *rkbuf,
779
+ rd_kafka_buf_t *request) {
780
+ int32_t TopicArrayCnt;
781
+ int i;
782
+ const int log_decode_errors = LOG_ERR;
783
+ rd_kafka_topic_t *rkt = NULL;
784
+ int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR;
785
+ rd_kafkap_Fetch_reply_tags_t FetchTags = RD_ZERO_INIT;
786
+ rd_bool_t has_fetch_tags = rd_false;
787
+
788
+ if (rd_kafka_buf_ApiVersion(request) >= 1) {
789
+ int32_t Throttle_Time;
790
+ rd_kafka_buf_read_i32(rkbuf, &Throttle_Time);
791
+
792
+ rd_kafka_op_throttle_time(rkb, rkb->rkb_rk->rk_rep,
793
+ Throttle_Time);
794
+ }
795
+
796
+ if (rd_kafka_buf_ApiVersion(request) >= 7) {
797
+ int32_t SessionId;
798
+ rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
799
+ rd_kafka_buf_read_i32(rkbuf, &SessionId);
800
+ }
801
+
802
+ rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX);
803
+ /* Verify that TopicArrayCnt seems to be in line with remaining size */
804
+ rd_kafka_buf_check_len(rkbuf,
805
+ TopicArrayCnt * (3 /*topic min size*/ +
806
+ 4 /*PartitionArrayCnt*/ + 4 +
807
+ 2 + 8 + 4 /*inner header*/));
808
+
809
+ if (rd_kafka_buf_ApiVersion(request) >= 12) {
810
+ has_fetch_tags = rd_true;
811
+ rd_kafkap_Fetch_reply_tags_set_topic_cnt(&FetchTags,
812
+ TopicArrayCnt);
813
+ }
814
+
815
+ for (i = 0; i < TopicArrayCnt; i++) {
816
+ rd_kafkap_str_t topic = RD_ZERO_INIT;
817
+ rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO;
818
+ int32_t PartitionArrayCnt;
819
+ int j;
820
+
821
+ if (rd_kafka_buf_ApiVersion(request) > 12) {
822
+ rd_kafka_buf_read_uuid(rkbuf, &topic_id);
823
+ rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk,
824
+ topic_id);
825
+ if (rkt)
826
+ topic = *rkt->rkt_topic;
827
+ } else {
828
+ rd_kafka_buf_read_str(rkbuf, &topic);
829
+ rkt = rd_kafka_topic_find0(rkb->rkb_rk, &topic);
830
+ }
831
+
832
+ rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt,
833
+ RD_KAFKAP_PARTITIONS_MAX);
834
+ if (rd_kafka_buf_ApiVersion(request) >= 12) {
835
+ rd_kafkap_Fetch_reply_tags_set_topic(
836
+ &FetchTags, i, topic_id, PartitionArrayCnt);
837
+ }
838
+
839
+ for (j = 0; j < PartitionArrayCnt; j++) {
840
+ if (rd_kafka_fetch_reply_handle_partition(
841
+ rkb, &topic, rkt, rkbuf, request, ErrorCode,
842
+ has_fetch_tags ? &FetchTags.Topics[i] : NULL,
843
+ has_fetch_tags
844
+ ? &FetchTags.Topics[i].Partitions[j]
845
+ : NULL))
846
+ goto err_parse;
847
+ }
848
+ if (has_fetch_tags &&
849
+ FetchTags.Topics[i].partitions_with_leader_change_cnt) {
850
+ FetchTags.topics_with_leader_change_cnt++;
851
+ }
852
+
853
+ if (rkt) {
854
+ rd_kafka_topic_destroy0(rkt);
855
+ rkt = NULL;
856
+ }
857
+ /* Topic Tags */
858
+ rd_kafka_buf_skip_tags(rkbuf);
859
+ }
860
+
861
+ /* Top level tags */
862
+ rd_kafka_buf_read_tags(rkbuf, rd_kafkap_Fetch_reply_tags_parse,
863
+ &FetchTags);
864
+
865
+ if (rd_kafka_buf_read_remain(rkbuf) != 0) {
866
+ rd_kafka_buf_parse_fail(rkbuf,
867
+ "Remaining data after message set "
868
+ "parse: %" PRIusz " bytes",
869
+ rd_kafka_buf_read_remain(rkbuf));
870
+ RD_NOTREACHED();
871
+ }
872
+ rd_kafka_handle_Fetch_metadata_update(rkb, &FetchTags);
873
+ rd_kafkap_Fetch_reply_tags_destroy(&FetchTags);
874
+
875
+ return 0;
876
+
877
+ err_parse:
878
+ if (rkt)
879
+ rd_kafka_topic_destroy0(rkt);
880
+ rd_kafkap_Fetch_reply_tags_destroy(&FetchTags);
881
+ rd_rkb_dbg(rkb, MSG, "BADMSG",
882
+ "Bad message (Fetch v%d): "
883
+ "is broker.version.fallback incorrectly set?",
884
+ (int)request->rkbuf_reqhdr.ApiVersion);
885
+ return rkbuf->rkbuf_err;
886
+ }
887
+
888
+ void rd_kafka_share_filter_acquired_records_and_update_ack_type(
889
+ rd_kafka_q_t *temp_fetchq,
890
+ rd_list_t *filtered_msgs,
891
+ const int64_t *FirstOffsets,
892
+ const int64_t *LastOffsets,
893
+ const int16_t *DeliveryCounts,
894
+ int32_t AcquiredRecordsArrayCnt) {
895
+
896
+ rd_kafka_op_t *rko;
897
+
898
+ /* Iterate through all messages in temp_fetchq and forward
899
+ * only those whose offset falls within an acquired range.
900
+ * Also set the ack type based on the op type. */
901
+ while ((rko = rd_kafka_q_pop(temp_fetchq, RD_POLL_NOWAIT, 0)) != NULL) {
902
+ int64_t rko_offset = rd_kafka_op_get_offset(rko);
903
+ rd_bool_t in_acquired_range = rd_false;
904
+ int16_t delivery_count;
905
+ int32_t range_idx;
906
+
907
+ /* Check if this message's offset is within any acquired range
908
+ */
909
+ for (range_idx = 0; range_idx < AcquiredRecordsArrayCnt;
910
+ range_idx++) {
911
+ if (rko_offset >= FirstOffsets[range_idx] &&
912
+ rko_offset <= LastOffsets[range_idx]) {
913
+ in_acquired_range = rd_true;
914
+ delivery_count = DeliveryCounts[range_idx];
915
+ break;
916
+ }
917
+ }
918
+
919
+ if (in_acquired_range) {
920
+ /* Set ack type based on op type */
921
+ rd_kafka_msg_t *rkm = NULL;
922
+ if (unlikely(rd_kafka_op_is_ctrl_msg(rko))) {
923
+ rd_kafka_op_destroy(rko);
924
+ continue;
925
+ }
926
+ if (rko->rko_type == RD_KAFKA_OP_FETCH) {
927
+ rkm = &rko->rko_u.fetch.rkm;
928
+ rkm->rkm_u.consumer.ack_type =
929
+ RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED;
930
+ rkm->rkm_u.consumer.delivery_count =
931
+ delivery_count;
932
+ } else if (rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) {
933
+ rkm = &rko->rko_u.err.rkm;
934
+ /* Set ack_type to RELEASE only if not already
935
+ * set by rd_kafka_share_msgset_err_ops()
936
+ * (which sets REJECT for CRC/unsupported
937
+ * errors, RELEASE for decompression errors). */
938
+ if (rkm->rkm_u.consumer.ack_type ==
939
+ RD_KAFKA_SHARE_INTERNAL_ACK_GAP ||
940
+ rkm->rkm_u.consumer.ack_type ==
941
+ RD_KAFKA_SHARE_INTERNAL_ACK_ACQUIRED)
942
+ rkm->rkm_u.consumer.ack_type =
943
+ RD_KAFKA_SHARE_INTERNAL_ACK_RELEASE;
944
+ }
945
+
946
+ /* Add to filtered messages list */
947
+ rd_list_add(filtered_msgs, rko);
948
+ } else {
949
+ /* Discard message not in any acquired range */
950
+ rd_kafka_op_destroy(rko);
951
+ }
952
+ }
953
+ }
954
+
955
+
956
+ /**
957
+ * @brief Comparator for sorting ops by offset.
958
+ *
959
+ * Note: rd_list_sort uses rd_list_cmp_trampoline which dereferences
960
+ * the pointers before calling this comparator, so we receive
961
+ * rd_kafka_op_t* directly, not rd_kafka_op_t**.
962
+ */
963
+ static int rd_kafka_op_offset_cmp(const void *_a, const void *_b) {
964
+ const rd_kafka_op_t *a = (const rd_kafka_op_t *)_a;
965
+ const rd_kafka_op_t *b = (const rd_kafka_op_t *)_b;
966
+ int64_t off_a = rd_kafka_op_get_offset(a);
967
+ int64_t off_b = rd_kafka_op_get_offset(b);
968
+
969
+ return (off_a > off_b) - (off_a < off_b);
970
+ }
971
+
972
+
973
+ /**
974
+ * @brief Check if op matches the given topic-partition.
975
+ *
976
+ * Compares the rktp (toppar) reference to ensure both topic and partition
977
+ * match, not just the partition number.
978
+ */
979
+ static rd_bool_t rd_kafka_op_matches_toppar(const rd_kafka_op_t *rko,
980
+ const rd_kafka_toppar_t *rktp) {
981
+ return rko->rko_rktp == rktp;
982
+ }
983
+
984
+ /**
985
+ * @brief Find the batch entry that contains the given offset.
986
+ *
987
+ * @returns The entry, or NULL if no entry contains the offset.
988
+ */
989
+ static rd_kafka_share_ack_batch_entry_t *
990
+ rd_kafka_share_find_entry_for_offset(rd_kafka_share_ack_batches_t *batches,
991
+ int64_t offset) {
992
+ rd_kafka_share_ack_batch_entry_t *entry;
993
+ int ei;
994
+
995
+ RD_LIST_FOREACH(entry, &batches->entries, ei) {
996
+ if (offset >= entry->start_offset &&
997
+ offset <= entry->end_offset)
998
+ return entry;
999
+ }
1000
+ return NULL;
1001
+ }
1002
+
1003
+ /**
1004
+ * @brief Get ack type from a message op (FETCH or CONSUMER_ERR).
1005
+ */
1006
+ static rd_kafka_share_internal_acknowledgement_type
1007
+ rd_kafka_share_ack_type_from_msg_op(rd_kafka_op_t *msg_rko) {
1008
+ rd_kafka_msg_t *rkm;
1009
+
1010
+ if (msg_rko->rko_type == RD_KAFKA_OP_FETCH)
1011
+ rkm = &msg_rko->rko_u.fetch.rkm;
1012
+ else
1013
+ rkm = &msg_rko->rko_u.err.rkm;
1014
+ return rkm->rkm_u.consumer.ack_type;
1015
+ }
1016
+
1017
+ /**
1018
+ * @brief Build share fetch response RKO with messages and update inflight acks.
1019
+ *
1020
+ * All entry types are pre-initialized to GAP in the reply handler. This
1021
+ * function overwrites ACQUIRED/REJECT for each offset that has a message
1022
+ * (RD_KAFKA_OP_FETCH or RD_KAFKA_OP_CONSUMER_ERR), and adds those messages
1023
+ * to the response. Moves inflight_acks batches to the response RKO.
1024
+ *
1025
+ * @param rkb Broker handle
1026
+ * @param filtered_msgs List of filtered message ops (with ack types already
1027
+ * set)
1028
+ * @param inflight_acks List of rd_kafka_share_ack_batches_t* (ownership
1029
+ * transferred to RKO)
1030
+ *
1031
+ * @returns New rko containing messages and inflight_acks, or NULL if empty
1032
+ */
1033
+ static rd_kafka_op_t *
1034
+ rd_kafka_share_build_response_rko(rd_kafka_broker_t *rkb,
1035
+ rd_list_t *filtered_msgs,
1036
+ rd_list_t *inflight_acks) {
1037
+
1038
+ rd_kafka_op_t *response_rko;
1039
+ rd_kafka_share_ack_batches_t *batches;
1040
+ int32_t msg_cnt = 0;
1041
+ int32_t inflight_acks_cnt = rd_list_cnt(inflight_acks);
1042
+ int pi, i;
1043
+ int total_msgs = rd_list_cnt(filtered_msgs);
1044
+ int msg_start_idx = 0;
1045
+ int64_t total_offsets = 0;
1046
+
1047
+ /* Create response rko */
1048
+ response_rko = rd_kafka_op_new(RD_KAFKA_OP_SHARE_FETCH_RESPONSE);
1049
+ response_rko->rko_rk = rkb->rkb_rk;
1050
+
1051
+ response_rko->rko_u.share_fetch_response.message_rkos =
1052
+ rd_list_new(0, NULL);
1053
+ response_rko->rko_u.share_fetch_response.inflight_acks =
1054
+ rd_list_new(0, rd_kafka_share_ack_batches_destroy_free);
1055
+
1056
+ /* Process each partition: set types for message offsets, add messages.
1057
+ * Messages in filtered_msgs are grouped by partition in the same order
1058
+ * as inflight_acks. Types are already GAP; we set ACQUIRED/REJECT only
1059
+ * where we have a message. */
1060
+ RD_LIST_FOREACH(batches, inflight_acks, pi) {
1061
+
1062
+ total_offsets += batches->response_acquired_offsets_count;
1063
+
1064
+ rd_kafka_topic_partition_private_t *parpriv =
1065
+ (rd_kafka_topic_partition_private_t *)
1066
+ batches->rktpar->_private;
1067
+ rd_kafka_toppar_t *rktp = parpriv->rktp;
1068
+
1069
+ rd_list_t partition_msgs;
1070
+ rd_list_init(&partition_msgs, 0, NULL);
1071
+
1072
+ while (msg_start_idx < total_msgs) {
1073
+ rd_kafka_op_t *candidate =
1074
+ rd_list_elem(filtered_msgs, msg_start_idx);
1075
+ if (!rd_kafka_op_matches_toppar(candidate, rktp))
1076
+ break;
1077
+ rd_list_add(&partition_msgs, candidate);
1078
+ msg_start_idx++;
1079
+ }
1080
+
1081
+ rd_list_sort(&partition_msgs, rd_kafka_op_offset_cmp);
1082
+
1083
+ int partition_msg_cnt = rd_list_cnt(&partition_msgs);
1084
+ for (i = 0; i < partition_msg_cnt; i++) {
1085
+ rd_kafka_op_t *msg_rko =
1086
+ rd_list_elem(&partition_msgs, i);
1087
+ int64_t offset = rd_kafka_op_get_offset(msg_rko);
1088
+ rd_kafka_share_ack_batch_entry_t *entry =
1089
+ rd_kafka_share_find_entry_for_offset(batches,
1090
+ offset);
1091
+
1092
+ if (unlikely(!entry)) {
1093
+ rd_rkb_dbg(
1094
+ rkb, FETCH, "SHAREFETCH",
1095
+ "No ack entry found for offset %" PRId64
1096
+ " on %s [%" PRId32 "], skipping",
1097
+ offset, batches->rktpar->topic,
1098
+ batches->rktpar->partition);
1099
+ rd_kafka_op_destroy(msg_rko);
1100
+ continue;
1101
+ }
1102
+
1103
+ entry->types[offset - entry->start_offset] =
1104
+ rd_kafka_share_ack_type_from_msg_op(msg_rko);
1105
+
1106
+ /**
1107
+ * The per message error ops (Decompression error, CRC
1108
+ * error, or MagicByte Errors are tracked in the same
1109
+ * list of messages as the successful messages.
1110
+ * TODO KIP-932: Check if we need a new op for record
1111
+ * level message errors: RD_KAFKA_OP_CONSUMER_MSG_ERR.
1112
+ */
1113
+ if (msg_rko->rko_type == RD_KAFKA_OP_FETCH ||
1114
+ msg_rko->rko_type == RD_KAFKA_OP_CONSUMER_ERR) {
1115
+ rd_list_add(
1116
+ response_rko->rko_u.share_fetch_response
1117
+ .message_rkos,
1118
+ msg_rko);
1119
+ msg_cnt++;
1120
+ }
1121
+ }
1122
+
1123
+ rd_list_destroy(&partition_msgs);
1124
+ rd_list_add(
1125
+ response_rko->rko_u.share_fetch_response.inflight_acks,
1126
+ rd_kafka_share_ack_batches_copy(batches));
1127
+ }
1128
+
1129
+ if (msg_cnt == 0 && inflight_acks_cnt == 0) {
1130
+ rd_kafka_op_destroy(response_rko);
1131
+ return NULL;
1132
+ }
1133
+
1134
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1135
+ "Built share fetch response rko with %d messages, "
1136
+ "%d gaps, and %d partitions",
1137
+ msg_cnt, (int32_t)(total_offsets - msg_cnt),
1138
+ inflight_acks_cnt);
1139
+
1140
+ return response_rko;
1141
+ }
1142
+
1143
+
1144
+ /**
1145
+ * @brief Handle a per-partition fetch error from a ShareFetch response.
1146
+ *
1147
+ * @locality broker thread
1148
+ */
1149
+ static void rd_kafka_share_fetch_reply_handle_partition_error(
1150
+ rd_kafka_broker_t *rkb,
1151
+ rd_kafka_toppar_t *rktp,
1152
+ const rd_kafkap_str_t *topic,
1153
+ int32_t partition,
1154
+ rd_kafka_resp_err_t err,
1155
+ const rd_kafkap_str_t *err_msg) {
1156
+
1157
+ /* TODO KIP-932: Verify whether the SURFACE-only arms below
1158
+ * (CORRUPT_MESSAGE, default unknown err) should also emit an
1159
+ * explicit warn-level log here. They currently rely on the
1160
+ * downstream OP_CONSUMER_ERR being surfaced to the app via
1161
+ * consume_batch. */
1162
+ /* TODO KIP-932: write test cases for each per-partition error
1163
+ * arm below once the mock cluster exposes a per-partition
1164
+ * error-injection API (e.g.
1165
+ * rd_kafka_mock_partition_push_share_fetch_error). Today only
1166
+ * the leader-change errors are exercised via
1167
+ * rd_kafka_mock_partition_set_leader; the remaining arms
1168
+ * (KAFKA_STORAGE_ERROR, OFFSET_NOT_AVAILABLE,
1169
+ * REPLICA_NOT_AVAILABLE, UNKNOWN_TOPIC_OR_PART,
1170
+ * UNKNOWN_TOPIC_ID, INCONSISTENT_TOPIC_ID,
1171
+ * TOPIC_AUTHORIZATION_FAILED, UNKNOWN_LEADER_EPOCH,
1172
+ * UNKNOWN_SERVER_ERROR, CORRUPT_MESSAGE, and the default)
1173
+ * have no deterministic mock trigger. */
1174
+ switch (err) {
1175
+ case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER:
1176
+ case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH:
1177
+ case RD_KAFKA_RESP_ERR_KAFKA_STORAGE_ERROR:
1178
+ case RD_KAFKA_RESP_ERR_OFFSET_NOT_AVAILABLE:
1179
+ case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE:
1180
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1181
+ "%.*s [%" PRId32
1182
+ "]: ShareFetch failed: %s: %.*s: "
1183
+ "triggering metadata refresh",
1184
+ RD_KAFKAP_STR_PR(topic), partition,
1185
+ rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
1186
+ rd_kafka_toppar_leader_unavailable(rktp, "sharefetch", err);
1187
+ break;
1188
+
1189
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
1190
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
1191
+ case RD_KAFKA_RESP_ERR_INCONSISTENT_TOPIC_ID:
1192
+ /* No per-partition recovery action; left for the next
1193
+ * metadata refresh / heartbeat reconciliation to resolve. */
1194
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1195
+ "%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s",
1196
+ RD_KAFKAP_STR_PR(topic), partition,
1197
+ rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
1198
+ break;
1199
+
1200
+ case RD_KAFKA_RESP_ERR_TOPIC_AUTHORIZATION_FAILED:
1201
+ rd_rkb_log(rkb, LOG_WARNING, "SHAREFETCH",
1202
+ "%.*s [%" PRId32 "]: Not authorized to read: %.*s",
1203
+ RD_KAFKAP_STR_PR(topic), partition,
1204
+ RD_KAFKAP_STR_PR(err_msg));
1205
+ rd_kafka_consumer_err(
1206
+ rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err,
1207
+ 0, NULL, rktp, RD_KAFKA_OFFSET_INVALID,
1208
+ "ShareFetch failed for %.*s [%" PRId32 "]: %.*s",
1209
+ RD_KAFKAP_STR_PR(topic), partition,
1210
+ RD_KAFKAP_STR_PR(err_msg));
1211
+ break;
1212
+
1213
+ case RD_KAFKA_RESP_ERR_UNKNOWN_LEADER_EPOCH:
1214
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1215
+ "%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s",
1216
+ RD_KAFKAP_STR_PR(topic), partition,
1217
+ rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
1218
+ break;
1219
+
1220
+ case RD_KAFKA_RESP_ERR_UNKNOWN:
1221
+ rd_rkb_log(rkb, LOG_WARNING, "SHAREFETCH",
1222
+ "%.*s [%" PRId32 "]: ShareFetch failed: %s: %.*s",
1223
+ RD_KAFKAP_STR_PR(topic), partition,
1224
+ rd_kafka_err2name(err), RD_KAFKAP_STR_PR(err_msg));
1225
+ break;
1226
+
1227
+ case RD_KAFKA_RESP_ERR_INVALID_MSG:
1228
+ rd_kafka_consumer_err(
1229
+ rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err,
1230
+ 0, NULL, rktp, RD_KAFKA_OFFSET_INVALID,
1231
+ "Encountered corrupt message when fetching "
1232
+ "topic-partition %.*s-%" PRId32 ": %.*s",
1233
+ RD_KAFKAP_STR_PR(topic), partition,
1234
+ RD_KAFKAP_STR_PR(err_msg));
1235
+ break;
1236
+
1237
+ default:
1238
+ rd_kafka_consumer_err(
1239
+ rkb->rkb_rk->rk_cgrp->rkcg_q, rd_kafka_broker_id(rkb), err,
1240
+ 0, NULL, rktp, RD_KAFKA_OFFSET_INVALID,
1241
+ "Unexpected error code %" PRId16
1242
+ " (%s) while fetching from topic-partition "
1243
+ "%.*s-%" PRId32 ": %.*s",
1244
+ (int16_t)err, rd_kafka_err2name(err),
1245
+ RD_KAFKAP_STR_PR(topic), partition,
1246
+ RD_KAFKAP_STR_PR(err_msg));
1247
+ break;
1248
+ }
1249
+ }
1250
+
1251
+
1252
+ /**
1253
+ * @brief Parse a partition from ShareFetch response and build inflight_acks.
1254
+ *
1255
+ * Creates rd_kafka_share_ack_batches_t directly with per-offset tracking.
1256
+ * All offsets are initialized to ACQUIRED state; types will be updated
1257
+ * later when matching with actual messages (GAP for missing, REJECT for
1258
+ * errors).
1259
+ *
1260
+ * @param rkb Broker handle
1261
+ * @param topic Topic name
1262
+ * @param topic_id Topic UUID
1263
+ * @param rkt Topic handle (possibly NULL)
1264
+ * @param rkbuf Response buffer
1265
+ * @param request Request buffer
1266
+ * @param temp_appq Queue to forward filtered messages
1267
+ * @param batches_out Output: inflight acks batches for this partition
1268
+ *
1269
+ * @returns Error code or RD_KAFKA_RESP_ERR_NO_ERROR on success
1270
+ *
1271
+ * TODO KIP-932: Check if we can reduce the number of args in this method.
1272
+ * Can we remove the topic field from the args, since it is a derived value
1273
+ * from rkt and initialize a topic variable inside this function?
1274
+ */
1275
+ static rd_kafka_resp_err_t rd_kafka_share_fetch_reply_handle_partition(
1276
+ rd_kafka_broker_t *rkb,
1277
+ const rd_kafkap_str_t *topic,
1278
+ rd_kafka_Uuid_t topic_id,
1279
+ rd_kafka_topic_t *rkt /*possibly NULL*/,
1280
+ rd_kafka_buf_t *rkbuf,
1281
+ rd_kafka_buf_t *request,
1282
+ rd_list_t *filtered_msgs,
1283
+ rd_kafka_share_ack_batches_t *batches_out,
1284
+ rd_list_t *request_ack_details) {
1285
+
1286
+ int32_t PartitionId;
1287
+ int16_t PartitionFetchErrorCode;
1288
+ rd_kafkap_str_t PartitionFetchErrorStr =
1289
+ RD_KAFKAP_STR_INITIALIZER_EMPTY;
1290
+ int16_t AcknowledgementErrorCode;
1291
+ rd_kafkap_str_t AcknowledgementErrorStr =
1292
+ RD_KAFKAP_STR_INITIALIZER_EMPTY;
1293
+ rd_kafkap_CurrentLeader_t CurrentLeader;
1294
+ int32_t MessageSetSize;
1295
+ rd_kafka_toppar_t *rktp = NULL;
1296
+ struct rd_kafka_toppar_ver tver;
1297
+ rd_slice_t save_slice;
1298
+ const int log_decode_errors = LOG_ERR;
1299
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
1300
+ int32_t AcquiredRecordsArrayCnt;
1301
+ int64_t *FirstOffsets = NULL;
1302
+ int64_t *LastOffsets = NULL;
1303
+ int16_t *DeliveryCounts = NULL;
1304
+ rd_kafka_q_t *temp_fetchq = rd_kafka_q_new(rkb->rkb_rk);
1305
+ int i;
1306
+ rd_bool_t is_sorted = rd_true;
1307
+ int64_t prev_end_offset = -1, size, j;
1308
+ rd_kafka_share_ack_batch_entry_t *entry;
1309
+ rd_kafka_topic_partition_private_t *parpriv;
1310
+ char *topic_str;
1311
+
1312
+ rd_kafka_buf_read_i32(rkbuf, &PartitionId); // Partition
1313
+ rd_kafka_buf_read_i16(rkbuf,
1314
+ &PartitionFetchErrorCode); // PartitionFetchError
1315
+ rd_kafka_buf_read_str(rkbuf, &PartitionFetchErrorStr); // ErrorString
1316
+ rd_kafka_buf_read_i16(
1317
+ rkbuf, &AcknowledgementErrorCode); // AcknowledgementError
1318
+ rd_kafka_buf_read_str(
1319
+ rkbuf, &AcknowledgementErrorStr); // AcknowledgementErrorString
1320
+
1321
+ /* Set the AcknowledgementErrorCode on the matching ack batch
1322
+ * in request_ack_details, so the main thread can map
1323
+ * per-partition ack errors to commit_sync results /
1324
+ * acknowledgement callback. PartitionFetchErrorCode is for
1325
+ * fetch-level errors and is handled separately. The partition
1326
+ * may be present in the response but absent from
1327
+ * request_ack_details (e.g. fetch-only partitions from
1328
+ * toppars_to_add) — silently skip those.
1329
+ *
1330
+ * Conditional on _IN_PROGRESS preserves any deliberately
1331
+ * pre-set err on the batch (e.g. INVALID_SHARE_SESSION_EPOCH
1332
+ * from the epoch-0 strip path) — those partitions were never
1333
+ * actually sent to the broker, so the broker's
1334
+ * AcknowledgementErrorCode (typically NO_ERROR) is not the
1335
+ * authoritative result for them. */
1336
+ if (request_ack_details) {
1337
+ rd_kafka_share_ack_batches_t *batch =
1338
+ rd_kafka_share_find_ack_batch_by_id(request_ack_details,
1339
+ topic_id, PartitionId);
1340
+ if (batch &&
1341
+ batch->rktpar->err == RD_KAFKA_RESP_ERR__IN_PROGRESS)
1342
+ batch->rktpar->err = AcknowledgementErrorCode;
1343
+ }
1344
+
1345
+ if (AcknowledgementErrorCode != RD_KAFKA_RESP_ERR_NO_ERROR) {
1346
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
1347
+ "ShareFetch response for %.*s [%" PRId32
1348
+ "]: AcknowledgementError %" PRId16 " (%s)",
1349
+ RD_KAFKAP_STR_PR(topic), PartitionId,
1350
+ AcknowledgementErrorCode,
1351
+ rd_kafka_err2str(AcknowledgementErrorCode));
1352
+ }
1353
+
1354
+ rd_kafka_buf_read_CurrentLeader(rkbuf,
1355
+ &CurrentLeader); // CurrentLeader
1356
+
1357
+ /* Compact Records Array */
1358
+ rd_kafka_buf_read_arraycnt(rkbuf, &MessageSetSize, -1);
1359
+
1360
+ if (unlikely(MessageSetSize < 0))
1361
+ rd_kafka_buf_parse_fail(
1362
+ rkbuf,
1363
+ "%.*s [%" PRId32 "]: invalid MessageSetSize %" PRId32,
1364
+ RD_KAFKAP_STR_PR(topic), PartitionId, MessageSetSize);
1365
+
1366
+ /* Look up topic+partition */
1367
+ if (likely(rkt != NULL)) {
1368
+ rd_kafka_topic_rdlock(rkt);
1369
+ rktp =
1370
+ rd_kafka_toppar_get(rkt, PartitionId, 0 /*no ua-on-miss*/);
1371
+ rd_kafka_topic_rdunlock(rkt);
1372
+ }
1373
+
1374
+ if (unlikely(!rkt || !rktp || PartitionFetchErrorCode)) {
1375
+ int64_t tmp_first, tmp_last;
1376
+ int16_t tmp_delivery;
1377
+
1378
+ if (!rkt) {
1379
+ /* TODO KIP-932: Recheck this branch once the
1380
+ * topic-recreate session-bookkeeping bug is fixed
1381
+ * (broker session may stop carrying old topic_ids
1382
+ * that no longer match any local rkt). */
1383
+ rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC",
1384
+ "Received Fetch response (error %hu) for "
1385
+ "unknown topic %.*s [%" PRId32 "]: ignoring",
1386
+ PartitionFetchErrorCode,
1387
+ RD_KAFKAP_STR_PR(topic), PartitionId);
1388
+ } else if (!rktp) {
1389
+ /* TODO KIP-932: Verify this handling against expected
1390
+ * behavior. librdkafka requires rktp to route records;
1391
+ * partitions with no local rktp are silently dropped.
1392
+ */
1393
+ rd_rkb_dbg(rkb, TOPIC, "UNKTOPIC",
1394
+ "Received Fetch response (error %hu) for "
1395
+ "unknown partition %.*s [%" PRId32
1396
+ "]: ignoring",
1397
+ PartitionFetchErrorCode,
1398
+ RD_KAFKAP_STR_PR(topic), PartitionId);
1399
+ } else {
1400
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1401
+ "%.*s [%" PRId32
1402
+ "]: per-partition fetch error %s",
1403
+ RD_KAFKAP_STR_PR(topic), PartitionId,
1404
+ rd_kafka_err2name(PartitionFetchErrorCode));
1405
+ rd_kafka_share_fetch_reply_handle_partition_error(
1406
+ rkb, rktp, topic, PartitionId,
1407
+ PartitionFetchErrorCode, &PartitionFetchErrorStr);
1408
+ }
1409
+
1410
+ rd_kafka_buf_skip(rkbuf, MessageSetSize);
1411
+ /* TODO KIP-932: Consider tracking total byte size of the
1412
+ * AcquiredRecords array in the protocol to allow a single
1413
+ * rd_kafka_buf_skip() here instead of per-entry parsing. */
1414
+ rd_kafka_buf_read_arraycnt(rkbuf, &AcquiredRecordsArrayCnt,
1415
+ -1); // AcquiredRecordsArrayCnt
1416
+ for (i = 0; i < AcquiredRecordsArrayCnt; i++) {
1417
+ rd_kafka_buf_read_i64(rkbuf,
1418
+ &tmp_first); // FirstOffset
1419
+ rd_kafka_buf_read_i64(rkbuf, &tmp_last); // LastOffset
1420
+ rd_kafka_buf_read_i16(rkbuf,
1421
+ &tmp_delivery); // DeliveryCount
1422
+ rd_kafka_buf_skip_tags(rkbuf); // AcquiredRecords tags
1423
+ }
1424
+ /* No records to track for this partition. Leave
1425
+ * batches_out->rktpar as NULL so the caller skips
1426
+ * adding this batch to inflight_acks. */
1427
+ rd_kafka_buf_skip_tags(rkbuf);
1428
+ goto done;
1429
+ }
1430
+
1431
+ tver.rktp = rktp;
1432
+ /* There is no versioning/barrier of the records/partitions in
1433
+ * share consumer case. */
1434
+ tver.version = 0;
1435
+
1436
+ if (MessageSetSize > 0) {
1437
+ /**
1438
+ * Parse MessageSet
1439
+ */
1440
+ if (!rd_slice_narrow_relative(&rkbuf->rkbuf_reader, &save_slice,
1441
+ (size_t)MessageSetSize))
1442
+ rd_kafka_buf_check_len(rkbuf, MessageSetSize);
1443
+
1444
+ /*
1445
+ * Parse messages
1446
+ */
1447
+ err = rd_kafka_share_msgset_parse(rkbuf, rktp, NULL, &tver,
1448
+ temp_fetchq);
1449
+
1450
+ rd_slice_widen(&rkbuf->rkbuf_reader, &save_slice);
1451
+ /* Continue with next partition regardless of
1452
+ * parse errors (which are partition-specific) */
1453
+
1454
+ /**
1455
+ * Only inner-record parsing errors (RD_KAFKA_RESP_ERR__BAD_MSG
1456
+ * / RD_KAFKA_RESP_ERR__UNDERFLOW) propagate out of
1457
+ * rd_kafka_share_msgset_parse for share consumers — per-batch
1458
+ * codec / CRC / unsupported-MagicByte failures are handled
1459
+ * inline by emitting per-offset RELEASE/REJECT ops and are
1460
+ * swallowed by the v2 reader (msgset_reader.c). When a true
1461
+ * parse error bubbles up the wire data is corrupt and we can no
1462
+ * longer trust the buffer position or any per-batch metadata.
1463
+ * Stop processing the rest of this ShareFetch response and
1464
+ * propagate the error to the caller, which aborts the whole
1465
+ * response handling.
1466
+ */
1467
+ if (err)
1468
+ goto done;
1469
+ }
1470
+
1471
+ rd_kafka_buf_read_arraycnt(rkbuf, &AcquiredRecordsArrayCnt,
1472
+ -1); // AcquiredRecordsArrayCnt
1473
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1474
+ "%.*s [%" PRId32 "] : AcquiredRecordsArrayCnt: %d",
1475
+ RD_KAFKAP_STR_PR(topic), PartitionId,
1476
+ AcquiredRecordsArrayCnt);
1477
+
1478
+ /* Allocate and initialize the topic partition */
1479
+ topic_str = RD_KAFKAP_STR_DUP(topic);
1480
+ batches_out->rktpar =
1481
+ rd_kafka_topic_partition_new(topic_str, PartitionId);
1482
+ rd_free(topic_str);
1483
+
1484
+ /* Allocate and fill the private structure */
1485
+ parpriv = rd_kafka_topic_partition_private_new();
1486
+ parpriv->rktp = rd_kafka_toppar_keep(rktp);
1487
+ parpriv->topic_id = topic_id;
1488
+ batches_out->rktpar->_private = parpriv;
1489
+
1490
+ /* Record the broker from which records were acquired so the
1491
+ * acknowledgement is sent back to the same broker. The wire
1492
+ * CurrentLeader hint is only set when the broker signals a
1493
+ * leader change, so we use the responding broker. */
1494
+ batches_out->response_leader_id = rkb->rkb_nodeid;
1495
+ batches_out->response_acquired_offsets_count = 0;
1496
+ /* Pre-allocate capacity without re-initializing the list.
1497
+ * batches_out->entries was already initialized by
1498
+ * rd_kafka_share_ack_batches_new() with the proper
1499
+ * entry destructor. */
1500
+ if (AcquiredRecordsArrayCnt > 0)
1501
+ rd_list_grow(&batches_out->entries, AcquiredRecordsArrayCnt);
1502
+
1503
+ if (AcquiredRecordsArrayCnt > 0) {
1504
+ FirstOffsets =
1505
+ rd_malloc(sizeof(*FirstOffsets) * AcquiredRecordsArrayCnt);
1506
+ LastOffsets =
1507
+ rd_malloc(sizeof(*LastOffsets) * AcquiredRecordsArrayCnt);
1508
+ DeliveryCounts = rd_malloc(sizeof(*DeliveryCounts) *
1509
+ AcquiredRecordsArrayCnt);
1510
+
1511
+ /**
1512
+ * TODO KIP-932: There could be an improvement where we
1513
+ * segregate the records while reading the FirstOffsets and
1514
+ * LastOffsets, so that we can avoid mallocs above and directly
1515
+ * create the batches and fill the entries.
1516
+ */
1517
+ for (i = 0; i < AcquiredRecordsArrayCnt; i++) {
1518
+ rd_kafka_buf_read_i64(rkbuf, &FirstOffsets[i]);
1519
+ rd_kafka_buf_read_i64(rkbuf, &LastOffsets[i]);
1520
+ rd_kafka_buf_read_i16(rkbuf, &DeliveryCounts[i]);
1521
+ rd_kafka_buf_skip_tags(rkbuf);
1522
+
1523
+ size = LastOffsets[i] - FirstOffsets[i] + 1;
1524
+
1525
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1526
+ "%.*s [%" PRId32
1527
+ "]: Acquired Records from offset %" PRId64
1528
+ " to %" PRId64 ", DeliveryCount %" PRId16,
1529
+ RD_KAFKAP_STR_PR(topic), PartitionId,
1530
+ FirstOffsets[i], LastOffsets[i],
1531
+ DeliveryCounts[i]);
1532
+
1533
+ /* Track sortedness: entries must be non-overlapping
1534
+ * and in ascending order by start_offset. */
1535
+ if (is_sorted && FirstOffsets[i] <= prev_end_offset)
1536
+ is_sorted = rd_false;
1537
+ prev_end_offset = LastOffsets[i];
1538
+
1539
+ entry = rd_kafka_share_ack_batch_entry_new(
1540
+ FirstOffsets[i], LastOffsets[i], (int32_t)size,
1541
+ DeliveryCounts[i]);
1542
+
1543
+ /* Initialize all offsets to GAP; build_response_rko
1544
+ * will set ACQUIRED/REJECT for offsets that have a
1545
+ * message. */
1546
+ for (j = 0; j < size; j++) {
1547
+ entry->types[j] =
1548
+ RD_KAFKA_SHARE_INTERNAL_ACK_GAP;
1549
+ }
1550
+
1551
+ rd_list_add(&batches_out->entries, entry);
1552
+ batches_out->response_acquired_offsets_count +=
1553
+ (int32_t)size;
1554
+ }
1555
+
1556
+ /* Mark as sorted to enable binary search in rd_list_find(). */
1557
+ if (is_sorted)
1558
+ batches_out->entries.rl_flags |= RD_LIST_F_SORTED;
1559
+
1560
+ /* Filter and forward messages in acquired ranges */
1561
+ rd_kafka_share_filter_acquired_records_and_update_ack_type(
1562
+ temp_fetchq, filtered_msgs, FirstOffsets, LastOffsets,
1563
+ DeliveryCounts, AcquiredRecordsArrayCnt);
1564
+ } else {
1565
+ /* No acquired ranges: drop everything */
1566
+ rd_kafka_op_t *rko;
1567
+ while ((rko = rd_kafka_q_pop(temp_fetchq, RD_POLL_NOWAIT, 0)))
1568
+ rd_kafka_op_destroy(rko);
1569
+ }
1570
+
1571
+ rd_kafka_buf_skip_tags(rkbuf); // Partition tags
1572
+
1573
+ goto done;
1574
+
1575
+ err_parse:
1576
+ err = rkbuf->rkbuf_err;
1577
+
1578
+ done:
1579
+ RD_IF_FREE(FirstOffsets, rd_free);
1580
+ RD_IF_FREE(LastOffsets, rd_free);
1581
+ RD_IF_FREE(DeliveryCounts, rd_free);
1582
+ RD_IF_FREE(temp_fetchq, rd_kafka_q_destroy_owner);
1583
+ if (rktp)
1584
+ rd_kafka_toppar_destroy(rktp); /* from toppar_get() */
1585
+ return err;
1586
+ }
1587
+
1588
+
1589
+ /**
1590
+ * Parses and handles a ShareFetch reply.
1591
+ * Returns 0 on success or an error code on failure.
1592
+ *
1593
+ * TODO KIP-932: Change return type to proper error with message. See
1594
+ * `rd_kafka_error_t *`.
1595
+ */
1596
+ static rd_kafka_resp_err_t
1597
+ rd_kafka_share_fetch_reply_handle(rd_kafka_broker_t *rkb,
1598
+ rd_kafka_buf_t *rkbuf,
1599
+ rd_kafka_buf_t *request,
1600
+ rd_kafka_op_t **response_rko_out) {
1601
+ int32_t TopicArrayCnt;
1602
+ int i;
1603
+ const int log_decode_errors = LOG_ERR;
1604
+ rd_kafka_topic_t *rkt = NULL;
1605
+ int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR;
1606
+ rd_kafkap_str_t ErrorStr = RD_KAFKAP_STR_INITIALIZER_EMPTY;
1607
+ int32_t AcquisitionLockTimeoutMs = 0;
1608
+ rd_kafkap_NodeEndpoints_t NodeEndpoints;
1609
+ NodeEndpoints.NodeEndpoints = NULL;
1610
+ NodeEndpoints.NodeEndpointCnt = 0;
1611
+ rd_list_t *filtered_msgs = NULL;
1612
+ rd_list_t *inflight_acks = NULL;
1613
+ rd_kafka_op_t *rko_orig = request->rkbuf_opaque;
1614
+ rd_kafka_op_t *response_rko = NULL;
1615
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
1616
+ rd_kafka_op_t *fetch_size_rko;
1617
+ int fetch_size_idx;
1618
+ int64_t total_fetch_size_bytes = 0;
1619
+
1620
+ rd_kafka_buf_read_throttle_time(rkbuf);
1621
+
1622
+ rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
1623
+ rd_kafka_buf_read_str(rkbuf, &ErrorStr);
1624
+
1625
+ if (ErrorCode) {
1626
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
1627
+ "ShareFetch response error %s: '%.*s'",
1628
+ rd_kafka_err2name(ErrorCode),
1629
+ RD_KAFKAP_STR_PR(&ErrorStr));
1630
+ return ErrorCode;
1631
+ }
1632
+
1633
+ /* Count this successful ShareFetch response for
1634
+ * consumer.share.fetch.manager.fetch.{total,rate}.
1635
+ * Fires only after top level error handling.
1636
+ *
1637
+ * TODO KIP-932: Revisit this sampling before GA. It fires here,
1638
+ * before the message set is parsed (rd_kafka_share_msgset_parse in
1639
+ * the partition loop below). If that parse fails with
1640
+ * RD_KAFKA_RESP_ERR__BAD_MSG / __UNDERFLOW the caller drops the
1641
+ * whole response, yet fetch.{total,rate} and fetch.latency have
1642
+ * already been counted here. */
1643
+ rd_atomic64_add(&rkb->rkb_rk->rk_telemetry.share_fetch_total, 1);
1644
+
1645
+ /* rkbuf_ts_sent is already the request RTT here */
1646
+ rd_avg_add(
1647
+ &rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_latency,
1648
+ request->rkbuf_ts_sent);
1649
+
1650
+ rd_kafka_buf_read_i32(rkbuf, &AcquisitionLockTimeoutMs);
1651
+
1652
+ rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX);
1653
+
1654
+ filtered_msgs = rd_list_new(0, NULL);
1655
+ inflight_acks = rd_list_new(0, rd_kafka_share_ack_batches_destroy_free);
1656
+
1657
+ for (i = 0; i < TopicArrayCnt; i++) {
1658
+ rd_kafkap_str_t topic = RD_ZERO_INIT;
1659
+ rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO;
1660
+ int32_t PartitionArrayCnt;
1661
+ int j;
1662
+
1663
+ rd_kafka_buf_read_uuid(rkbuf, &topic_id);
1664
+ rkt = rd_kafka_topic_find_by_topic_id(rkb->rkb_rk, topic_id);
1665
+ if (rkt)
1666
+ topic = *rkt->rkt_topic;
1667
+
1668
+ rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt,
1669
+ RD_KAFKAP_PARTITIONS_MAX);
1670
+
1671
+ for (j = 0; j < PartitionArrayCnt; j++) {
1672
+ rd_kafka_share_ack_batches_t *batches =
1673
+ rd_kafka_share_ack_batches_new_empty();
1674
+
1675
+ err = rd_kafka_share_fetch_reply_handle_partition(
1676
+ rkb, &topic, topic_id, rkt, rkbuf, request,
1677
+ filtered_msgs, batches,
1678
+ rko_orig->rko_u.share_fetch.ack_details);
1679
+
1680
+ /*
1681
+ * Only inner-record parsing errors
1682
+ * (RD_KAFKA_RESP_ERR__BAD_MSG /
1683
+ * RD_KAFKA_RESP_ERR__UNDERFLOW) reach this point —
1684
+ * per-batch CRC / decompression / MagicByte failures
1685
+ * are handled inline inside
1686
+ * rd_kafka_share_msgset_parse by emitting per-offset
1687
+ * RELEASE/REJECT ops and are swallowed, and
1688
+ * per-partition broker errors are handled by
1689
+ * rd_kafka_share_fetch_reply_handle_partition_error
1690
+ * before MessageSet parsing begins. A parsing error
1691
+ * means the wire data is corrupt and the buffer
1692
+ * position can no longer be trusted for subsequent
1693
+ * partitions, so abort the whole response.
1694
+ */
1695
+ if (err) {
1696
+ rd_kafka_share_ack_batches_destroy(batches);
1697
+ goto done;
1698
+ }
1699
+
1700
+ /* Skip unknown topics - don't add to inflight_acks */
1701
+ if (batches->rktpar == NULL) {
1702
+ rd_kafka_share_ack_batches_destroy(batches);
1703
+ continue;
1704
+ }
1705
+
1706
+ if (rd_list_cnt(&batches->entries) > 0) {
1707
+ rd_list_add(inflight_acks, batches);
1708
+ } else {
1709
+ rd_kafka_share_ack_batches_destroy(batches);
1710
+ }
1711
+ }
1712
+
1713
+ if (rkt) {
1714
+ rd_kafka_topic_destroy0(rkt);
1715
+ rkt = NULL;
1716
+ }
1717
+ /* Topic Tags */
1718
+ rd_kafka_buf_skip_tags(rkbuf);
1719
+ }
1720
+
1721
+ rd_kafka_buf_read_NodeEndpoints(rkbuf, &NodeEndpoints);
1722
+
1723
+ /* Sum per-record wire bytes across acquired records for
1724
+ * consumer.share.fetch.manager.fetch.size.{avg,max} and
1725
+ * .bytes.consumed.{total,rate}. */
1726
+ RD_LIST_FOREACH(fetch_size_rko, filtered_msgs, fetch_size_idx) {
1727
+ if (fetch_size_rko->rko_type == RD_KAFKA_OP_FETCH) {
1728
+ rd_kafka_msg_t *rkm = &fetch_size_rko->rko_u.fetch.rkm;
1729
+ total_fetch_size_bytes +=
1730
+ (int64_t)rkm->rkm_u.consumer.wire_size;
1731
+ }
1732
+ }
1733
+ /* Update bytes per fetch metrics */
1734
+ rd_avg_add(&rkb->rkb_telemetry.rd_avg_current.rkb_avg_share_fetch_size,
1735
+ total_fetch_size_bytes);
1736
+
1737
+ /* Update total bytes consumed */
1738
+ rd_atomic64_add(&rkb->rkb_rk->rk_telemetry.share_bytes_consumed_total,
1739
+ total_fetch_size_bytes);
1740
+
1741
+ /* Build response rko with messages and inflight_acks */
1742
+ response_rko = rd_kafka_share_build_response_rko(rkb, filtered_msgs,
1743
+ inflight_acks);
1744
+
1745
+ rd_list_destroy(inflight_acks);
1746
+ inflight_acks = NULL;
1747
+ rd_list_destroy(filtered_msgs);
1748
+ filtered_msgs = NULL;
1749
+
1750
+ /* Top level tags */
1751
+ rd_kafka_buf_skip_tags(rkbuf);
1752
+
1753
+ /* Return response_rko to the caller instead of enqueueing here.
1754
+ * The caller enqueues it on rkcg_q AFTER sending the reply to
1755
+ * rk_ops, ensuring the main thread processes the reply (and
1756
+ * resets share_fetch_more_records) before the app thread can
1757
+ * wake up, poll again, and enqueue a new FANOUT. */
1758
+ *response_rko_out = response_rko;
1759
+
1760
+ /* Signal main thread whether records were fetched */
1761
+ if (rko_orig)
1762
+ rko_orig->rko_u.share_fetch.records_fetched =
1763
+ response_rko ? rd_true : rd_false;
1764
+
1765
+ RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free);
1766
+ RD_IF_FREE(rkt, rd_kafka_topic_destroy0);
1767
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
1768
+
1769
+ err_parse:
1770
+ err = rkbuf->rkbuf_err;
1771
+
1772
+ done:
1773
+ /* Lists are NULLed on the success path after their inline destroy;
1774
+ * here they're only non-NULL on the error path. filtered_msgs uses a
1775
+ * NULL destructor, so any ops accumulated from earlier partitions
1776
+ * before err_parse fired must be destroyed explicitly. */
1777
+ RD_IF_FREE(inflight_acks, rd_list_destroy);
1778
+ if (filtered_msgs) {
1779
+ rd_kafka_op_t *rko;
1780
+ int li;
1781
+ RD_LIST_FOREACH(rko, filtered_msgs, li)
1782
+ rd_kafka_op_destroy(rko);
1783
+ rd_list_destroy(filtered_msgs);
1784
+ }
1785
+
1786
+ RD_IF_FREE(response_rko, rd_kafka_op_destroy);
1787
+ *response_rko_out = NULL;
1788
+
1789
+ if (rkt)
1790
+ rd_kafka_topic_destroy0(rkt);
1791
+ rd_rkb_dbg(rkb, MSG, "BADMSG", "Bad message (ShareFetch v%d): %s",
1792
+ (int)request->rkbuf_reqhdr.ApiVersion,
1793
+ rd_kafka_err2str(err));
1794
+ return err;
1795
+ }
1796
+
1797
+
1798
+ /**
1799
+ * @brief Reset the share fetch session for a broker.
1800
+ *
1801
+ * Called when the broker returns SHARE_SESSION_NOT_FOUND,
1802
+ * INVALID_SHARE_SESSION_EPOCH, or SHARE_SESSION_LIMIT_REACHED,
1803
+ * indicating the session is lost or cannot be created.
1804
+ * Resets the epoch to 0 and moves all toppars_in_session back
1805
+ * to toppars_to_add so the next request re-establishes the
1806
+ * full session.
1807
+ *
1808
+ * @param rkb Broker whose session is being reset.
1809
+ *
1810
+ * @locality broker thread
1811
+ */
1812
+ static void rd_kafka_broker_session_reset(rd_kafka_broker_t *rkb) {
1813
+ rd_kafka_toppar_t *rktp;
1814
+ int i;
1815
+
1816
+ rd_rkb_dbg(
1817
+ rkb, FETCH, "SHARESESSION",
1818
+ "Resetting share session: epoch %" PRId32
1819
+ " -> 0, "
1820
+ "moving %d toppars_in_session to toppars_to_add",
1821
+ rkb->rkb_share_fetch_session.epoch,
1822
+ rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session));
1823
+
1824
+ rkb->rkb_share_fetch_session.epoch = 0;
1825
+
1826
+ /* Remove toppars_to_forget from toppars_in_session — these
1827
+ * were pending removal and should not be re-added to the
1828
+ * new session. */
1829
+ if (rkb->rkb_share_fetch_session.toppars_to_forget) {
1830
+ rd_kafka_toppar_t *forget_rktp;
1831
+
1832
+ RD_LIST_FOREACH(forget_rktp,
1833
+ rkb->rkb_share_fetch_session.toppars_to_forget,
1834
+ i) {
1835
+ rktp = rd_list_remove(
1836
+ rkb->rkb_share_fetch_session.toppars_in_session,
1837
+ forget_rktp);
1838
+ if (rktp)
1839
+ rd_kafka_toppar_destroy(rktp);
1840
+ }
1841
+
1842
+ rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_forget);
1843
+ rkb->rkb_share_fetch_session.toppars_to_forget = NULL;
1844
+ }
1845
+
1846
+ /* Move remaining toppars_in_session to toppars_to_add so they
1847
+ * get sent as new additions in the next request (epoch 0). */
1848
+ if (!rkb->rkb_share_fetch_session.toppars_to_add) {
1849
+ rkb->rkb_share_fetch_session.toppars_to_add =
1850
+ rkb->rkb_share_fetch_session.toppars_in_session;
1851
+ rkb->rkb_share_fetch_session.toppars_in_session =
1852
+ rd_list_new(0, rd_kafka_toppar_destroy_free);
1853
+ } else {
1854
+ while ((rktp = rd_list_pop(
1855
+ rkb->rkb_share_fetch_session.toppars_in_session))) {
1856
+ if (!rd_list_find(
1857
+ rkb->rkb_share_fetch_session.toppars_to_add,
1858
+ rktp, rd_list_cmp_ptr))
1859
+ rd_list_add(
1860
+ rkb->rkb_share_fetch_session.toppars_to_add,
1861
+ rktp);
1862
+ else
1863
+ rd_kafka_toppar_destroy(rktp);
1864
+ }
1865
+ }
1866
+ }
1867
+
1868
+ static void rd_kafka_broker_session_update_epoch(rd_kafka_broker_t *rkb) {
1869
+ int32_t prev_epoch = rkb->rkb_share_fetch_session.epoch;
1870
+ if (prev_epoch == -1) {
1871
+ rd_rkb_dbg(
1872
+ rkb, FETCH, "SHARESESSION",
1873
+ "Not updating next epoch for -1 as it should be -1 again.");
1874
+ return;
1875
+ }
1876
+ if (prev_epoch == INT32_MAX)
1877
+ rkb->rkb_share_fetch_session.epoch = 1;
1878
+ else
1879
+ rkb->rkb_share_fetch_session.epoch++;
1880
+ rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
1881
+ "share-fetch session epoch %" PRId32 " -> %" PRId32
1882
+ " (wrap=%s)",
1883
+ prev_epoch, rkb->rkb_share_fetch_session.epoch,
1884
+ prev_epoch == INT32_MAX ? "yes" : "no");
1885
+ }
1886
+
1887
+ static void rd_kafka_broker_session_add_partition_to_toppars_in_session(
1888
+ rd_kafka_broker_t *rkb,
1889
+ rd_kafka_toppar_t *rktp) {
1890
+ if (rd_list_find(rkb->rkb_share_fetch_session.toppars_in_session, rktp,
1891
+ rd_list_cmp_ptr)) {
1892
+ rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
1893
+ "%s [%" PRId32 "]: already in ShareFetch session",
1894
+ rktp->rktp_rkt->rkt_topic->str,
1895
+ rktp->rktp_partition);
1896
+ return;
1897
+ }
1898
+ rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
1899
+ "%s [%" PRId32 "]: adding to ShareFetch session",
1900
+ rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition);
1901
+ rd_kafka_toppar_keep(rktp);
1902
+ rd_list_add(rkb->rkb_share_fetch_session.toppars_in_session, rktp);
1903
+ }
1904
+
1905
+ void rd_kafka_broker_session_remove_partition_from_toppars_in_session(
1906
+ rd_kafka_broker_t *rkb,
1907
+ rd_kafka_toppar_t *rktp) {
1908
+ rd_kafka_toppar_t *removed_rktp;
1909
+ removed_rktp = rd_list_remove(
1910
+ rkb->rkb_share_fetch_session.toppars_in_session, rktp);
1911
+ if (removed_rktp) {
1912
+ rd_kafka_toppar_destroy(removed_rktp);
1913
+ rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
1914
+ "%s [%" PRId32 "]: removed from ShareFetch session",
1915
+ rktp->rktp_rkt->rkt_topic->str,
1916
+ rktp->rktp_partition);
1917
+ } else {
1918
+ rd_rkb_dbg(rkb, FETCH, "SHARESESSION",
1919
+ "%s [%" PRId32 "]: not found in ShareFetch session",
1920
+ rktp->rktp_rkt->rkt_topic->str,
1921
+ rktp->rktp_partition);
1922
+ }
1923
+ }
1924
+
1925
+ static void
1926
+ rd_kafka_broker_session_update_toppars_in_session(rd_kafka_broker_t *rkb,
1927
+ rd_kafka_toppar_t *rktp,
1928
+ rd_bool_t add) {
1929
+ if (add)
1930
+ rd_kafka_broker_session_add_partition_to_toppars_in_session(
1931
+ rkb, rktp);
1932
+ else
1933
+ rd_kafka_broker_session_remove_partition_from_toppars_in_session(
1934
+ rkb, rktp);
1935
+ }
1936
+
1937
+ static void rd_kafka_broker_session_update_toppars_list(
1938
+ rd_kafka_broker_t *rkb,
1939
+ rd_list_t **request_toppars_ptr,
1940
+ rd_list_t **toppars_to_remove_from_ptr,
1941
+ rd_bool_t add) {
1942
+ size_t i;
1943
+ rd_kafka_toppar_t *rktp, *removed_rktp;
1944
+ rd_list_t *request_toppars = *request_toppars_ptr;
1945
+ rd_list_t *toppars_to_remove_from = *toppars_to_remove_from_ptr;
1946
+
1947
+ if (request_toppars == NULL || rd_list_cnt(request_toppars) == 0)
1948
+ return;
1949
+
1950
+ rd_rkb_dbg(
1951
+ rkb, FETCH, "SHARESESSION",
1952
+ "%d toppars being %s the session:", rd_list_cnt(request_toppars),
1953
+ add ? "added to" : "removed from");
1954
+
1955
+ RD_LIST_FOREACH(rktp, request_toppars, i) {
1956
+ rd_kafka_broker_session_update_toppars_in_session(rkb, rktp,
1957
+ add);
1958
+ if (toppars_to_remove_from) {
1959
+ removed_rktp =
1960
+ rd_list_remove(toppars_to_remove_from, rktp);
1961
+ if (removed_rktp) {
1962
+ rd_kafka_toppar_destroy(
1963
+ removed_rktp); /* from partitions list */
1964
+ if (rd_list_empty(toppars_to_remove_from)) {
1965
+ rd_list_destroy(toppars_to_remove_from);
1966
+ toppars_to_remove_from = NULL;
1967
+ *toppars_to_remove_from_ptr = NULL;
1968
+ }
1969
+ }
1970
+ }
1971
+ }
1972
+ rd_list_destroy(request_toppars);
1973
+ *request_toppars_ptr = NULL;
1974
+ }
1975
+
1976
+ static void
1977
+ rd_kafka_broker_session_update_added_partitions(rd_kafka_broker_t *rkb) {
1978
+ rd_kafka_broker_session_update_toppars_list(
1979
+ rkb, &rkb->rkb_share_fetch_session.adding_toppars,
1980
+ &rkb->rkb_share_fetch_session.toppars_to_add, rd_true);
1981
+ }
1982
+
1983
+ static void
1984
+ rd_kafka_broker_session_update_removed_partitions(rd_kafka_broker_t *rkb) {
1985
+ rd_kafka_broker_session_update_toppars_list(
1986
+ rkb, &rkb->rkb_share_fetch_session.forgetting_toppars,
1987
+ &rkb->rkb_share_fetch_session.toppars_to_forget, rd_false);
1988
+ }
1989
+
1990
+ static void rd_kafka_broker_session_update_partitions(rd_kafka_broker_t *rkb) {
1991
+ rd_rkb_dbg(
1992
+ rkb, FETCH, "SHARESESSION",
1993
+ "applying session updates: adding_toppars=%d "
1994
+ "forgetting_toppars=%d toppars_in_session=%d epoch=%" PRId32,
1995
+ rkb->rkb_share_fetch_session.adding_toppars
1996
+ ? rd_list_cnt(rkb->rkb_share_fetch_session.adding_toppars)
1997
+ : 0,
1998
+ rkb->rkb_share_fetch_session.forgetting_toppars
1999
+ ? rd_list_cnt(rkb->rkb_share_fetch_session.forgetting_toppars)
2000
+ : 0,
2001
+ rkb->rkb_share_fetch_session.toppars_in_session
2002
+ ? rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session)
2003
+ : 0,
2004
+ rkb->rkb_share_fetch_session.epoch);
2005
+ rd_kafka_broker_session_update_added_partitions(rkb);
2006
+ rd_kafka_broker_session_update_removed_partitions(rkb);
2007
+ }
2008
+
2009
+
2010
+ /**
2011
+ * Update ShareFetch session state after a Fetch or ShareFetch response.
2012
+ * TODO KIP-932: Improve efficiency of this function.
2013
+ */
2014
+ static void rd_kafka_broker_session_update(rd_kafka_broker_t *rkb) {
2015
+ rd_kafka_broker_session_update_epoch(rkb);
2016
+ rd_kafka_broker_session_update_partitions(rkb);
2017
+ }
2018
+
2019
+ /**
2020
+ * @brief Whether \p err is a per-partition ShareAcknowledge error
2021
+ * whose CurrentLeader hint should trigger an inline metadata
2022
+ * update.
2023
+ */
2024
+ static rd_bool_t
2025
+ rd_kafka_share_ack_err_is_leader_change(rd_kafka_resp_err_t err) {
2026
+ switch (err) {
2027
+ case RD_KAFKA_RESP_ERR_NOT_LEADER_OR_FOLLOWER:
2028
+ case RD_KAFKA_RESP_ERR_FENCED_LEADER_EPOCH:
2029
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
2030
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
2031
+ return rd_true;
2032
+ default:
2033
+ return rd_false;
2034
+ }
2035
+ }
2036
+
2037
+ /**
2038
+ * @brief Parse a ShareAcknowledge response.
2039
+ *
2040
+ * ShareAcknowledge response contains per-partition error codes for
2041
+ * acknowledgement results. Returns the top-level error code and sets
2042
+ * the err field on each matching batch in \p ack_details.
2043
+ *
2044
+ * TODO KIP-932: Consider using rd_kafka_error_t instead of
2045
+ * rd_kafka_resp_err_t for the per-partition result and
2046
+ * acknowledgement callback.
2047
+ *
2048
+ * @param rkb Broker handle.
2049
+ * @param rkbuf Response buffer.
2050
+ * @param request Original request buffer.
2051
+ * @param ack_details List of acknowledgement batches sent in the
2052
+ * request. Each matched batch's rktpar->err is
2053
+ * updated with the partition-level error code
2054
+ * from the response.
2055
+ *
2056
+ * @returns Top-level error code.
2057
+ * @locality broker thread
2058
+ */
2059
+ static rd_kafka_resp_err_t
2060
+ rd_kafka_share_acknowledge_reply_handle(rd_kafka_broker_t *rkb,
2061
+ rd_kafka_buf_t *rkbuf,
2062
+ rd_kafka_buf_t *request,
2063
+ rd_list_t *ack_details) {
2064
+ int32_t TopicArrayCnt;
2065
+ int i;
2066
+ const int log_decode_errors = LOG_ERR;
2067
+ int16_t ErrorCode = RD_KAFKA_RESP_ERR_NO_ERROR;
2068
+ rd_kafkap_str_t ErrorStr = RD_KAFKAP_STR_INITIALIZER_EMPTY;
2069
+ rd_kafkap_NodeEndpoints_t NodeEndpoints;
2070
+ rd_list_t *leader_changes = NULL;
2071
+ NodeEndpoints.NodeEndpoints = NULL;
2072
+ NodeEndpoints.NodeEndpointCnt = 0;
2073
+
2074
+ rd_kafka_buf_read_throttle_time(rkbuf);
2075
+
2076
+ rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
2077
+ rd_kafka_buf_read_str(rkbuf, &ErrorStr);
2078
+
2079
+ if (ErrorCode) {
2080
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
2081
+ "ShareAcknowledge response error %s: '%.*s'",
2082
+ rd_kafka_err2name(ErrorCode),
2083
+ RD_KAFKAP_STR_PR(&ErrorStr));
2084
+ return ErrorCode;
2085
+ }
2086
+
2087
+ rd_kafka_buf_read_arraycnt(rkbuf, &TopicArrayCnt, RD_KAFKAP_TOPICS_MAX);
2088
+
2089
+ for (i = 0; i < TopicArrayCnt; i++) {
2090
+ rd_kafka_Uuid_t topic_id = RD_KAFKA_UUID_ZERO;
2091
+ int32_t PartitionArrayCnt;
2092
+ int j;
2093
+
2094
+ rd_kafka_buf_read_uuid(rkbuf, &topic_id);
2095
+
2096
+ rd_kafka_buf_read_arraycnt(rkbuf, &PartitionArrayCnt,
2097
+ RD_KAFKAP_PARTITIONS_MAX);
2098
+
2099
+ for (j = 0; j < PartitionArrayCnt; j++) {
2100
+ int32_t Partition;
2101
+ int16_t PartErrorCode;
2102
+ rd_kafkap_str_t PartErrorStr;
2103
+ rd_kafkap_CurrentLeader_t CurrentLeader;
2104
+ rd_kafka_share_ack_batches_t *batch;
2105
+ rd_kafkap_share_leader_change_t *lc;
2106
+
2107
+ rd_kafka_buf_read_i32(rkbuf, &Partition);
2108
+ rd_kafka_buf_read_i16(rkbuf, &PartErrorCode);
2109
+ rd_kafka_buf_read_str(rkbuf, &PartErrorStr);
2110
+
2111
+ /* CurrentLeader */
2112
+ rd_kafka_buf_read_CurrentLeader(rkbuf, &CurrentLeader);
2113
+
2114
+ batch = rd_kafka_share_find_ack_batch_by_id(
2115
+ ack_details, topic_id, Partition);
2116
+ if (batch) {
2117
+ /* Conditional on _IN_PROGRESS as a
2118
+ * defensive safety net — the broker's
2119
+ * per-partition PartErrorCode is the
2120
+ * authoritative result for partitions
2121
+ * actually sent in the request. */
2122
+ if (batch->rktpar->err ==
2123
+ RD_KAFKA_RESP_ERR__IN_PROGRESS)
2124
+ batch->rktpar->err = PartErrorCode;
2125
+ } else {
2126
+ rd_rkb_log(
2127
+ rkb, LOG_ERR, "SHAREACK",
2128
+ "Invalid partition %" PRId32
2129
+ " received in ShareAcknowledge response",
2130
+ Partition);
2131
+ }
2132
+
2133
+ if (PartErrorCode) {
2134
+ /* TODO KIP-932: write test cases for each
2135
+ * per-partition ShareAcknowledge error code
2136
+ * once the mock cluster exposes a
2137
+ * per-partition error-injection API for
2138
+ * ShareAcknowledge responses. Today only
2139
+ * NOT_LEADER_OR_FOLLOWER (via
2140
+ * rd_kafka_mock_partition_set_leader) is
2141
+ * deterministically reachable; the other
2142
+ * leader-change errors
2143
+ * (FENCED_LEADER_EPOCH,
2144
+ * UNKNOWN_TOPIC_OR_PART,
2145
+ * UNKNOWN_TOPIC_ID),
2146
+ * INVALID_RECORD_STATE,
2147
+ * INVALID_REQUEST, KAFKA_STORAGE_ERROR
2148
+ * and the inline-leader-update path have
2149
+ * no deterministic mock trigger. */
2150
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
2151
+ "ShareAcknowledge partition %" PRId32
2152
+ " error %s: '%.*s'",
2153
+ Partition,
2154
+ rd_kafka_err2name(PartErrorCode),
2155
+ RD_KAFKAP_STR_PR(&PartErrorStr));
2156
+
2157
+ if (rd_kafka_share_ack_err_is_leader_change(
2158
+ PartErrorCode) &&
2159
+ CurrentLeader.LeaderId != -1 &&
2160
+ CurrentLeader.LeaderEpoch != -1) {
2161
+ if (!leader_changes)
2162
+ leader_changes =
2163
+ rd_list_new(0, rd_free);
2164
+ lc = rd_calloc(1, sizeof(*lc));
2165
+ lc->topic_id = topic_id;
2166
+ lc->partition = Partition;
2167
+ lc->current_leader = CurrentLeader;
2168
+ rd_list_add(leader_changes, lc);
2169
+ }
2170
+ }
2171
+
2172
+ /* Partition tags */
2173
+ rd_kafka_buf_skip_tags(rkbuf);
2174
+ }
2175
+
2176
+ /* Topic tags */
2177
+ rd_kafka_buf_skip_tags(rkbuf);
2178
+ }
2179
+
2180
+ rd_kafka_buf_read_NodeEndpoints(rkbuf, &NodeEndpoints);
2181
+
2182
+ /* Top level tags */
2183
+ rd_kafka_buf_skip_tags(rkbuf);
2184
+
2185
+ rd_kafkap_share_leader_changes_apply(rkb, leader_changes,
2186
+ &NodeEndpoints);
2187
+
2188
+ RD_IF_FREE(leader_changes, rd_list_destroy);
2189
+ RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free);
2190
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
2191
+
2192
+ err_parse:
2193
+ RD_IF_FREE(leader_changes, rd_list_destroy);
2194
+ RD_IF_FREE(NodeEndpoints.NodeEndpoints, rd_free);
2195
+ rd_rkb_dbg(rkb, MSG, "BADMSG",
2196
+ "Bad ShareAcknowledge response (v%d): parse error",
2197
+ (int)request->rkbuf_reqhdr.ApiVersion);
2198
+ return rkbuf->rkbuf_err;
2199
+ }
2200
+
2201
+
2202
+ /**
2203
+ * @brief ShareAcknowledge response handling callback.
2204
+ *
2205
+ * @locality broker thread (or any thread if err == __DESTROY).
2206
+ */
2207
+ static void rd_kafka_broker_share_acknowledge_reply(rd_kafka_t *rk,
2208
+ rd_kafka_broker_t *rkb,
2209
+ rd_kafka_resp_err_t err,
2210
+ rd_kafka_buf_t *reply,
2211
+ rd_kafka_buf_t *request,
2212
+ void *opaque) {
2213
+ rd_kafka_op_t *rko_orig = opaque;
2214
+
2215
+ /* Parse and handle the response (unless the request errored).
2216
+ * The parser writes per-partition err on matching batches in
2217
+ * ack_details (only on batches still at the _IN_PROGRESS
2218
+ * sentinel as a defensive safety net). DESTROY case falls
2219
+ * through here and is handled by the generic top-level error
2220
+ * path below.
2221
+ *
2222
+ * If the parser ran and succeeded (err remains 0), convert
2223
+ * any batch still at _IN_PROGRESS to INVALID_RECORD_STATE —
2224
+ * those partitions were sent in the request but missing from
2225
+ * the response.
2226
+ *
2227
+ * If the parser was skipped or failed (err != 0), leave
2228
+ * batches at _IN_PROGRESS so the helper at the end can
2229
+ * propagate the top-level err to them. */
2230
+ if (!err && reply) {
2231
+ err = rd_kafka_share_acknowledge_reply_handle(
2232
+ rkb, reply, request,
2233
+ rko_orig->rko_u.share_fetch.ack_details);
2234
+ if (!err && rko_orig->rko_u.share_fetch.ack_details) {
2235
+ rd_kafka_share_ack_batches_t *batch;
2236
+ int i;
2237
+ RD_LIST_FOREACH(
2238
+ batch, rko_orig->rko_u.share_fetch.ack_details, i) {
2239
+ if (batch->rktpar->err ==
2240
+ RD_KAFKA_RESP_ERR__IN_PROGRESS)
2241
+ batch->rktpar->err =
2242
+ RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE;
2243
+ }
2244
+ }
2245
+ }
2246
+
2247
+ rd_kafka_broker_session_update(rkb);
2248
+
2249
+ if (unlikely(err)) {
2250
+ rd_rkb_log(rkb, LOG_INFO, "SHAREACK",
2251
+ "ShareAcknowledge reply error: %s",
2252
+ rd_kafka_err2str(err));
2253
+ switch (err) {
2254
+ case RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND:
2255
+ case RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH:
2256
+ case RD_KAFKA_RESP_ERR__TRANSPORT:
2257
+ case RD_KAFKA_RESP_ERR__TIMED_OUT:
2258
+ /* __TRANSPORT means connection is already dead.
2259
+ * On __TIMED_OUT the connection is torn down by the
2260
+ * request-timeout scan (socket.max.fails is forced to
2261
+ * 1 for share consumers and cannot be changed) and the
2262
+ * broker reconnects via the share-serve persistent
2263
+ * connection driver; here we only reset the session so
2264
+ * it re-establishes at epoch 0. */
2265
+ rd_kafka_broker_session_reset(rkb);
2266
+ break;
2267
+
2268
+ case RD_KAFKA_RESP_ERR__BAD_MSG:
2269
+ case RD_KAFKA_RESP_ERR__UNDERFLOW:
2270
+ /* Wire-level parse failure: response envelope or
2271
+ * an inner MessageSet/record was malformed or
2272
+ * truncated. Indicates broker bug, on-the-wire
2273
+ * corruption, or version mismatch — log loudly so
2274
+ * the user notices, since the LOG_INFO top-level
2275
+ * log can be easy to miss in production output. */
2276
+ rd_rkb_log(
2277
+ rkb, LOG_ERR, "SHAREACK",
2278
+ "ShareAcknowledge response parse failure: %s "
2279
+ "(ApiVersion %hd) — broker bug, wire "
2280
+ "corruption, or version mismatch; "
2281
+ "this response is being dropped",
2282
+ rd_kafka_err2str(err),
2283
+ request->rkbuf_reqhdr.ApiVersion);
2284
+ break;
2285
+
2286
+ default:
2287
+ /* No retry for ShareAcknowledge RPC-level
2288
+ * errors. The error semantics (partition
2289
+ * moved, session state lost, etc.) make a
2290
+ * blind retry unlikely to succeed and risk
2291
+ * duplicating side effects. */
2292
+ break;
2293
+ }
2294
+ }
2295
+
2296
+ if (rko_orig->rko_u.share_fetch.should_leave)
2297
+ rd_kafka_broker_share_fetch_session_clear(rkb);
2298
+
2299
+ /* ack_details is owned by the op and freed by the op destructor
2300
+ * after the main thread has processed the reply. */
2301
+ rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(rko_orig,
2302
+ err);
2303
+ }
2304
+
2305
+
2306
+ /**
2307
+ * @broker ShareFetchResponse handling.
2308
+ *
2309
+ * @locality broker thread (or any thread if err == __DESTROY).
2310
+ */
2311
+ static void rd_kafka_broker_share_fetch_reply(rd_kafka_t *rk,
2312
+ rd_kafka_broker_t *rkb,
2313
+ rd_kafka_resp_err_t err,
2314
+ rd_kafka_buf_t *reply,
2315
+ rd_kafka_buf_t *request,
2316
+ void *opaque) {
2317
+
2318
+ rd_kafka_op_t *rko_orig = opaque;
2319
+ rd_kafka_op_t *response_rko = NULL;
2320
+
2321
+ /* Parse the response only if the network/broker layer didn't
2322
+ * report an error. If err is set (e.g. __TRANSPORT,
2323
+ * __TIMED_OUT, __DESTROY), the reply buffer is unusable so
2324
+ * we skip parsing. The parser writes per-partition err on
2325
+ * matching batches in ack_details (only on batches still at
2326
+ * the _IN_PROGRESS sentinel, so any deliberately pre-set
2327
+ * value such as INVALID_SHARE_SESSION_EPOCH is preserved).
2328
+ * The DESTROY/network error case falls through to the
2329
+ * generic top-level error path below.
2330
+ *
2331
+ * If the parser ran and succeeded (err remains 0), convert
2332
+ * any batch still at _IN_PROGRESS to INVALID_RECORD_STATE —
2333
+ * those partitions were sent in the request but missing from
2334
+ * the response.
2335
+ *
2336
+ * If the parser was skipped or failed (err != 0), leave
2337
+ * batches at _IN_PROGRESS so the helper at the end can
2338
+ * propagate the top-level err to them. */
2339
+ if (!err && reply) {
2340
+ err = rd_kafka_share_fetch_reply_handle(rkb, reply, request,
2341
+ &response_rko);
2342
+ if (!err && rko_orig->rko_u.share_fetch.ack_details) {
2343
+ rd_kafka_share_ack_batches_t *batch;
2344
+ int i;
2345
+ RD_LIST_FOREACH(
2346
+ batch, rko_orig->rko_u.share_fetch.ack_details, i) {
2347
+ if (batch->rktpar->err ==
2348
+ RD_KAFKA_RESP_ERR__IN_PROGRESS)
2349
+ batch->rktpar->err =
2350
+ RD_KAFKA_RESP_ERR_INVALID_RECORD_STATE;
2351
+ }
2352
+ }
2353
+ }
2354
+
2355
+ rd_kafka_broker_session_update(rkb);
2356
+
2357
+ if (unlikely(err)) {
2358
+ rd_rkb_log(rkb, LOG_INFO, "SHAREFETCH",
2359
+ "ShareFetch reply error: %s", rd_kafka_err2str(err));
2360
+ switch (err) {
2361
+ case RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND:
2362
+ case RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH:
2363
+ case RD_KAFKA_RESP_ERR_SHARE_SESSION_LIMIT_REACHED:
2364
+ case RD_KAFKA_RESP_ERR__TRANSPORT:
2365
+ case RD_KAFKA_RESP_ERR__TIMED_OUT:
2366
+ /* Session is invalid, lost, cannot be created,
2367
+ * or connection/request failed.
2368
+ * Reset session state so the next request
2369
+ * re-establishes a new session (epoch 0).
2370
+ * __TRANSPORT means connection is already dead.
2371
+ * On __TIMED_OUT the connection is torn down by the
2372
+ * request-timeout scan (socket.max.fails is forced to
2373
+ * 1 for share consumers and cannot be changed) and the
2374
+ * broker reconnects via the share-serve persistent
2375
+ * connection driver. */
2376
+ rd_kafka_broker_session_reset(rkb);
2377
+ break;
2378
+
2379
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID: {
2380
+ char tmp[128];
2381
+ rd_snprintf(tmp, sizeof(tmp), "ShareFetch failed: %s",
2382
+ rd_kafka_err2str(err));
2383
+ rd_kafka_metadata_refresh_known_topics(
2384
+ rkb->rkb_rk, NULL, rd_false /*!force*/, tmp);
2385
+ break;
2386
+ }
2387
+
2388
+ case RD_KAFKA_RESP_ERR__BAD_MSG:
2389
+ case RD_KAFKA_RESP_ERR__UNDERFLOW:
2390
+ /* Wire-level parse failure: response envelope or
2391
+ * an inner MessageSet/record was malformed or
2392
+ * truncated. Indicates broker bug, on-the-wire
2393
+ * corruption, or version mismatch — log loudly so
2394
+ * the user notices, since the LOG_INFO top-level
2395
+ * log can be easy to miss in production output. */
2396
+ rd_rkb_log(rkb, LOG_ERR, "SHAREFETCH",
2397
+ "ShareFetch response parse failure: %s "
2398
+ "(ApiVersion %hd) — broker bug, wire "
2399
+ "corruption, or version mismatch; "
2400
+ "this response is being dropped",
2401
+ rd_kafka_err2str(err),
2402
+ request->rkbuf_reqhdr.ApiVersion);
2403
+ break;
2404
+
2405
+ default:
2406
+ /* No error-specific handling at the request
2407
+ * level. Non-session top-level errors are
2408
+ * treated as transient — the main thread
2409
+ * retries by selecting another broker. */
2410
+ break;
2411
+ }
2412
+
2413
+ /* There is no retry for ShareFetch RPC at the broker
2414
+ * thread level. */
2415
+ }
2416
+
2417
+ /* ack_details is owned by the op and freed by the op destructor
2418
+ * after the main thread has processed the reply. */
2419
+ rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(rko_orig,
2420
+ err);
2421
+
2422
+ /* Enqueue the response for the app thread AFTER sending the
2423
+ * reply to the main thread. This ensures the main thread
2424
+ * processes the reply (resetting share_fetch_more_records)
2425
+ * before the app thread wakes up and enqueues a new FANOUT. */
2426
+ if (response_rko)
2427
+ rd_kafka_q_enq(rkb->rkb_rk->rk_cgrp->rkcg_q, response_rko);
2428
+ }
2429
+
2430
+ /**
2431
+ * @broker FetchResponse handling.
2432
+ *
2433
+ * @locality broker thread (or any thread if err == __DESTROY).
2434
+ */
2435
+ static void rd_kafka_broker_fetch_reply(rd_kafka_t *rk,
2436
+ rd_kafka_broker_t *rkb,
2437
+ rd_kafka_resp_err_t err,
2438
+ rd_kafka_buf_t *reply,
2439
+ rd_kafka_buf_t *request,
2440
+ void *opaque) {
2441
+
2442
+ if (err == RD_KAFKA_RESP_ERR__DESTROY)
2443
+ return; /* Terminating */
2444
+
2445
+ rd_kafka_assert(rkb->rkb_rk, rkb->rkb_fetching > 0);
2446
+ rkb->rkb_fetching = 0;
2447
+
2448
+ /* Parse and handle the messages (unless the request errored) */
2449
+ if (!err && reply)
2450
+ err = rd_kafka_fetch_reply_handle(rkb, reply, request);
2451
+
2452
+ if (unlikely(err)) {
2453
+ char tmp[128];
2454
+
2455
+ rd_rkb_dbg(rkb, MSG, "FETCH", "Fetch reply: %s",
2456
+ rd_kafka_err2str(err));
2457
+ switch (err) {
2458
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART:
2459
+ case RD_KAFKA_RESP_ERR_LEADER_NOT_AVAILABLE:
2460
+ case RD_KAFKA_RESP_ERR_NOT_LEADER_FOR_PARTITION:
2461
+ case RD_KAFKA_RESP_ERR_BROKER_NOT_AVAILABLE:
2462
+ case RD_KAFKA_RESP_ERR_REPLICA_NOT_AVAILABLE:
2463
+ case RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_ID:
2464
+ /* Request metadata information update */
2465
+ rd_snprintf(tmp, sizeof(tmp), "FetchRequest failed: %s",
2466
+ rd_kafka_err2str(err));
2467
+ rd_kafka_metadata_refresh_known_topics(
2468
+ rkb->rkb_rk, NULL, rd_true /*force*/, tmp);
2469
+ /* FALLTHRU */
2470
+
2471
+ case RD_KAFKA_RESP_ERR__TRANSPORT:
2472
+ case RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT:
2473
+ case RD_KAFKA_RESP_ERR__MSG_TIMED_OUT:
2474
+ /* The fetch is already intervalled from
2475
+ * consumer_serve() so dont retry. */
2476
+ break;
2477
+
2478
+ default:
2479
+ break;
2480
+ }
2481
+
2482
+ rd_kafka_broker_fetch_backoff(rkb, err);
2483
+ /* FALLTHRU */
2484
+ }
2485
+ }
2486
+
2487
+ /**
2488
+ * @brief Check if any toppars have a zero topic id.
2489
+ *
2490
+ */
2491
+ static rd_bool_t can_use_topic_ids(rd_kafka_broker_t *rkb) {
2492
+ rd_kafka_toppar_t *rktp = rkb->rkb_active_toppar_next;
2493
+ do {
2494
+ if (RD_KAFKA_UUID_IS_ZERO(rktp->rktp_rkt->rkt_topic_id))
2495
+ return rd_false;
2496
+ } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp,
2497
+ rktp_activelink)) !=
2498
+ rkb->rkb_active_toppar_next);
2499
+
2500
+ return rd_true;
2501
+ }
2502
+
2503
+
2504
+ /**
2505
+ * @brief Sum the total acknowledgement entries and records across all
2506
+ * per-partition batches in \p ack_details.
2507
+ *
2508
+ * @param ack_details List of rd_kafka_share_ack_batches_t, or NULL.
2509
+ * @param entries_out Set to the total number of ack entries (offset ranges).
2510
+ * @param records_out Set to the total number of acknowledged records
2511
+ * (sum of entry->size across all entries).
2512
+ */
2513
+ static void rd_kafka_share_ack_details_totals(rd_list_t *ack_details,
2514
+ int *entries_out,
2515
+ int64_t *records_out) {
2516
+ rd_kafka_share_ack_batches_t *batches;
2517
+ rd_kafka_share_ack_batch_entry_t *entry;
2518
+ int k, j;
2519
+ int entries = 0;
2520
+ int64_t records = 0;
2521
+
2522
+ if (ack_details) {
2523
+ RD_LIST_FOREACH(batches, ack_details, k) {
2524
+ entries += rd_list_cnt(&batches->entries);
2525
+ RD_LIST_FOREACH(entry, &batches->entries, j) {
2526
+ records += entry->size;
2527
+ }
2528
+ }
2529
+ }
2530
+
2531
+ *entries_out = entries;
2532
+ *records_out = records;
2533
+ }
2534
+
2535
+
2536
+ void rd_kafka_ShareFetchRequest(rd_kafka_broker_t *rkb,
2537
+ const rd_kafkap_str_t *group_id,
2538
+ const rd_kafkap_str_t *member_id,
2539
+ int32_t share_session_epoch,
2540
+ int32_t wait_max_ms,
2541
+ int32_t min_bytes,
2542
+ int32_t max_bytes,
2543
+ int32_t max_records,
2544
+ int32_t batch_size,
2545
+ rd_list_t *toppars_to_add,
2546
+ rd_list_t *toppars_to_forget,
2547
+ rd_kafka_op_t *rko_orig,
2548
+ rd_ts_t now) {
2549
+ rd_kafka_toppar_t *rktp;
2550
+ rd_kafka_buf_t *rkbuf;
2551
+ int cnt = 0;
2552
+ size_t of_TopicArrayCnt = 0;
2553
+ int TopicArrayCnt = 0;
2554
+ size_t of_PartitionArrayCnt = 0;
2555
+ int PartitionArrayCnt = 0;
2556
+ rd_kafka_topic_t *rkt_last = NULL;
2557
+ int16_t ApiVersion = 0;
2558
+ size_t rkbuf_size = 0;
2559
+ int toppars_to_add_cnt =
2560
+ toppars_to_add ? rd_list_cnt(toppars_to_add) : 0;
2561
+ int i;
2562
+ rd_list_t *ack_details =
2563
+ rko_orig ? rko_orig->rko_u.share_fetch.ack_details : NULL;
2564
+ int ack_details_cnt = ack_details ? rd_list_cnt(ack_details) : 0;
2565
+ /* TODO KIP-932: Ensure there is no intersection between toppars_to_add
2566
+ * and ack_details. A toppar should not appear in both lists. */
2567
+ int total_ack_entries = 0;
2568
+ int64_t total_ack_records = 0;
2569
+ int toppars_to_forget_cnt =
2570
+ toppars_to_forget ? rd_list_cnt(toppars_to_forget) : 0;
2571
+ rd_bool_t is_fetching_messages = max_records > 0 ? rd_true : rd_false;
2572
+ /* FirstOffset + LastOffset + AcknowledgementType per ack entry */
2573
+ size_t acknowledgement_size = 8 + 8 + 1;
2574
+
2575
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
2576
+ "toppars_to_add_cnt=%d, ack_details_cnt=%d, "
2577
+ "total_ack_entries=%d, toppars_to_forget_cnt=%d, "
2578
+ "is_fetching_messages=%d",
2579
+ toppars_to_add_cnt, ack_details_cnt, total_ack_entries,
2580
+ toppars_to_forget_cnt, is_fetching_messages);
2581
+
2582
+ /* Calculate buffer size */
2583
+ if (group_id)
2584
+ rkbuf_size += RD_KAFKAP_STR_SIZE(group_id);
2585
+ if (member_id)
2586
+ rkbuf_size += RD_KAFKAP_STR_SIZE(member_id);
2587
+ /* ShareSessionEpoch + WaitMaxMs + MinBytes + MaxBytes + MaxRecords +
2588
+ * BatchSize + TopicArrayCnt*/
2589
+ rkbuf_size += 4 + 4 + 4 + 4 + 4 + 4 + 4;
2590
+ /* N x (topic id + partition id) for topics to add */
2591
+ rkbuf_size += (toppars_to_add_cnt * (32 + 4));
2592
+ /* M x (topic id + partition id) for ack details partitions +
2593
+ * E x acknowledgement entries */
2594
+
2595
+ rkbuf_size += (ack_details_cnt * (32 + 4));
2596
+ /* Sum total ack entries and total records across all
2597
+ * ack_details batches. */
2598
+ rd_kafka_share_ack_details_totals(ack_details, &total_ack_entries,
2599
+ &total_ack_records);
2600
+ /* Accumulate piggybacked record-level acknowledgements for
2601
+ * consumer.share.fetch.manager.acknowledgements.send.{total,rate}. */
2602
+ if (total_ack_records > 0)
2603
+ rd_atomic64_add(
2604
+ &rkb->rkb_rk->rk_telemetry.acknowledgements_send_total,
2605
+ total_ack_records);
2606
+
2607
+ rkbuf_size += (total_ack_entries * acknowledgement_size);
2608
+
2609
+ /* F x (topic id + partition id) for topics to forget */
2610
+ rkbuf_size += (toppars_to_forget_cnt * (32 + 4));
2611
+
2612
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(
2613
+ rkb, RD_KAFKAP_ShareFetch, 1, 1, NULL);
2614
+
2615
+ rkbuf = rd_kafka_buf_new_flexver_request(rkb, RD_KAFKAP_ShareFetch, 1,
2616
+ rkbuf_size, rd_true);
2617
+
2618
+ if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2)
2619
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
2620
+ RD_KAFKA_FEATURE_MSGVER2);
2621
+ else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1)
2622
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
2623
+ RD_KAFKA_FEATURE_MSGVER1);
2624
+ else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME)
2625
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
2626
+ RD_KAFKA_FEATURE_THROTTLETIME);
2627
+
2628
+ /* GroupId */
2629
+ rd_kafka_buf_write_kstr(rkbuf, group_id);
2630
+
2631
+ /* MemberId */
2632
+ rd_kafka_buf_write_kstr(rkbuf, member_id);
2633
+
2634
+ // printf(" ---------------------------------------
2635
+ // rd_kafka_ShareFetchRequest: member_id=%.*s\n",
2636
+ // RD_KAFKAP_STR_PR(member_id));
2637
+
2638
+ /* ShareSessionEpoch */
2639
+ rd_kafka_buf_write_i32(rkbuf, share_session_epoch);
2640
+
2641
+ /* WaitMaxMs */
2642
+ rd_kafka_buf_write_i32(rkbuf, wait_max_ms);
2643
+
2644
+ /* MinBytes */
2645
+ rd_kafka_buf_write_i32(rkbuf, min_bytes);
2646
+
2647
+ /* MaxBytes */
2648
+ rd_kafka_buf_write_i32(rkbuf, max_bytes);
2649
+
2650
+ /* MaxRecords */
2651
+ rd_kafka_buf_write_i32(rkbuf, max_records);
2652
+
2653
+ /* BatchSize */
2654
+ rd_kafka_buf_write_i32(rkbuf, batch_size);
2655
+
2656
+ /* Write zero TopicArrayCnt but store pointer for later update */
2657
+ of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
2658
+
2659
+ /* TODO KIP-932: Ensure toppars_to_add and ack_details don't have
2660
+ * common rktps. A toppar should only appear in one list.
2661
+ * Also merge toppars_to_add and ack_details as both can have
2662
+ * the same topic but different partitions. */
2663
+
2664
+ /* Write toppars_to_add: new toppars being added to session
2665
+ * (no acknowledgements for newly added toppars) */
2666
+ if (toppars_to_add) {
2667
+ /* TODO KIP-932: This condition will cause partitions of same
2668
+ topics to be inside single instance of the topic as
2669
+ toppars_to_add is not sorted. Eg: T1 0, T1 1, T2 0, T1 3, T1
2670
+ 5, T2 1 will translate to T1 (0,1), T2 (0), T1 (3, 5), T2
2671
+ (1) instead it should be T1 (0,1,3,5) T2(0,1) Fix this. */
2672
+ RD_LIST_FOREACH(rktp, toppars_to_add, i) {
2673
+ if (rkt_last != rktp->rktp_rkt) {
2674
+ if (rkt_last != NULL) {
2675
+ /* Update PartitionArrayCnt */
2676
+ rd_kafka_buf_finalize_arraycnt(
2677
+ rkbuf, of_PartitionArrayCnt,
2678
+ PartitionArrayCnt);
2679
+ /* Topic tags */
2680
+ rd_kafka_buf_write_tags_empty(rkbuf);
2681
+ }
2682
+
2683
+ rd_kafka_topic_rdlock(rktp->rktp_rkt);
2684
+ /* Topic ID */
2685
+ rd_kafka_buf_write_uuid(
2686
+ rkbuf, &rktp->rktp_rkt->rkt_topic_id);
2687
+ rd_kafka_topic_rdunlock(rktp->rktp_rkt);
2688
+
2689
+ TopicArrayCnt++;
2690
+ rkt_last = rktp->rktp_rkt;
2691
+ /* Partition count */
2692
+ of_PartitionArrayCnt =
2693
+ rd_kafka_buf_write_arraycnt_pos(rkbuf);
2694
+ PartitionArrayCnt = 0;
2695
+ }
2696
+
2697
+ PartitionArrayCnt++;
2698
+
2699
+ /* Partition */
2700
+ rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition);
2701
+
2702
+ /* No acknowledgements for newly added toppars */
2703
+ rd_kafka_buf_write_arraycnt(rkbuf, 0);
2704
+
2705
+ /* Partition tags */
2706
+ rd_kafka_buf_write_tags_empty(rkbuf);
2707
+
2708
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
2709
+ "Share Fetch adding topic %.*s [%" PRId32
2710
+ "]",
2711
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
2712
+ rktp->rktp_partition);
2713
+
2714
+ cnt++;
2715
+ }
2716
+
2717
+ /* Finalize last topic from toppars_to_add */
2718
+ if (rkt_last != NULL) {
2719
+ rd_kafka_buf_finalize_arraycnt(
2720
+ rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
2721
+ rd_kafka_buf_write_tags_empty(rkbuf);
2722
+ }
2723
+ }
2724
+
2725
+ /* Write ack_details: toppars with acknowledgement batches.
2726
+ * Uses rktpar directly (no rktp/lock needed). */
2727
+ if (ack_details) {
2728
+ rd_kafka_Uuid_t *topic_id_last = NULL;
2729
+ rd_kafka_share_ack_batches_t *batches;
2730
+ rd_kafka_topic_partition_private_t *parpriv;
2731
+ rd_kafka_share_ack_batch_entry_t *entry;
2732
+ int k, m, entries_cnt;
2733
+
2734
+ /* TODO KIP-932: This condition will cause partitions of same
2735
+ topics to be inside single instance of the topic as
2736
+ ack_details is not sorted. Eg: T1 0, T1 1, T2 0, T1 3, T1
2737
+ 5, T2 1 will translate to T1 (0,1), T2 (0), T1 (3, 5), T2
2738
+ (1) instead it should be T1 (0,1,3,5) T2(0,1) Fix this. */
2739
+ PartitionArrayCnt = 0;
2740
+
2741
+ RD_LIST_FOREACH(batches, ack_details, k) {
2742
+ /* TODO KIP-932: Ensure rktpar and rktp are in sync,
2743
+ * leader has not changed in between, and rktp is
2744
+ * present before reaching this point.
2745
+ * Also ensure batches->entries are not empty. */
2746
+ /* _private should always be present here as it was
2747
+ * set when building the ack batches from the
2748
+ * inflight map. */
2749
+ parpriv = (rd_kafka_topic_partition_private_t *)
2750
+ batches->rktpar->_private;
2751
+ rd_dassert(parpriv != NULL);
2752
+
2753
+ if (topic_id_last == NULL ||
2754
+ rd_kafka_Uuid_cmp(*topic_id_last,
2755
+ parpriv->topic_id) != 0) {
2756
+ if (topic_id_last != NULL) {
2757
+ /* Update PartitionArrayCnt */
2758
+ rd_kafka_buf_finalize_arraycnt(
2759
+ rkbuf, of_PartitionArrayCnt,
2760
+ PartitionArrayCnt);
2761
+ /* Topic tags */
2762
+ rd_kafka_buf_write_tags_empty(rkbuf);
2763
+ }
2764
+
2765
+ /* Topic ID */
2766
+ rd_kafka_buf_write_uuid(rkbuf,
2767
+ &parpriv->topic_id);
2768
+
2769
+ TopicArrayCnt++;
2770
+ topic_id_last = &parpriv->topic_id;
2771
+ /* Partition count */
2772
+ of_PartitionArrayCnt =
2773
+ rd_kafka_buf_write_arraycnt_pos(rkbuf);
2774
+ PartitionArrayCnt = 0;
2775
+ }
2776
+
2777
+ PartitionArrayCnt++;
2778
+
2779
+ /* Partition */
2780
+ rd_kafka_buf_write_i32(rkbuf,
2781
+ batches->rktpar->partition);
2782
+
2783
+ /* Write acknowledgement batches */
2784
+ entries_cnt = rd_list_cnt(&batches->entries);
2785
+ rd_kafka_buf_write_arraycnt(rkbuf, entries_cnt);
2786
+
2787
+ RD_LIST_FOREACH(entry, &batches->entries, m) {
2788
+ /* FirstOffset */
2789
+ rd_kafka_buf_write_i64(rkbuf,
2790
+ entry->start_offset);
2791
+ /* LastOffset */
2792
+ rd_kafka_buf_write_i64(rkbuf,
2793
+ entry->end_offset);
2794
+ /* AcknowledgeTypes */
2795
+ rd_kafka_buf_write_arraycnt(rkbuf, 1);
2796
+ rd_kafka_buf_write_i8(rkbuf,
2797
+ (int8_t)entry->types[0]);
2798
+ /* Acknowledgement tags */
2799
+ rd_kafka_buf_write_tags_empty(rkbuf);
2800
+ }
2801
+
2802
+ /* Partition tags */
2803
+ rd_kafka_buf_write_tags_empty(rkbuf);
2804
+
2805
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
2806
+ "Share Fetch ack for topic %s [%" PRId32
2807
+ "] with %d entries",
2808
+ batches->rktpar->topic,
2809
+ batches->rktpar->partition, entries_cnt);
2810
+
2811
+ cnt++;
2812
+ }
2813
+
2814
+ /* Finalize last topic from ack_details */
2815
+ if (topic_id_last != NULL) {
2816
+ rd_kafka_buf_finalize_arraycnt(
2817
+ rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
2818
+ rd_kafka_buf_write_tags_empty(rkbuf);
2819
+ }
2820
+ }
2821
+
2822
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
2823
+ "Share Fetch Request (epoch %" PRId32
2824
+ ") with %d toppars on %d topics",
2825
+ rkb->rkb_share_fetch_session.epoch, cnt, TopicArrayCnt);
2826
+
2827
+ /* Update TopicArrayCnt */
2828
+ rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
2829
+
2830
+ if (toppars_to_forget) {
2831
+ TopicArrayCnt = 0;
2832
+ PartitionArrayCnt = 0;
2833
+ rkt_last = NULL;
2834
+ /* Write zero TopicArrayCnt but store pointer for later update
2835
+ */
2836
+ of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
2837
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH", "Forgetting %d toppars",
2838
+ toppars_to_forget_cnt);
2839
+ RD_LIST_FOREACH(rktp, toppars_to_forget, i) {
2840
+ /* TODO KIP-932: This condition will cause partitions of
2841
+ same topics to be inside single instance of the topic as
2842
+ toppars_to_forget is not sorted. Eg: T1 0, T1 1, T2 0,
2843
+ T1 3, T1 5, T2 1 will translate to T1 (0,1), T2 (0),
2844
+ T1 (3, 5), T2 (1) instead it should be T1 (0,1,3,5)
2845
+ T2(0,1) Fix this. */
2846
+ if (rkt_last != rktp->rktp_rkt) {
2847
+ if (rkt_last != NULL) {
2848
+ /* Update PartitionArrayCnt */
2849
+ rd_kafka_buf_finalize_arraycnt(
2850
+ rkbuf, of_PartitionArrayCnt,
2851
+ PartitionArrayCnt);
2852
+ /* Topic tags */
2853
+ rd_kafka_buf_write_tags_empty(rkbuf);
2854
+ }
2855
+
2856
+ rd_kafka_topic_rdlock(rktp->rktp_rkt);
2857
+ /* Topic ID */
2858
+ rd_kafka_buf_write_uuid(
2859
+ rkbuf, &rktp->rktp_rkt->rkt_topic_id);
2860
+ rd_kafka_topic_rdunlock(rktp->rktp_rkt);
2861
+
2862
+ TopicArrayCnt++;
2863
+ rkt_last = rktp->rktp_rkt;
2864
+ /* Partition count */
2865
+ of_PartitionArrayCnt =
2866
+ rd_kafka_buf_write_arraycnt_pos(rkbuf);
2867
+ PartitionArrayCnt = 0;
2868
+ }
2869
+
2870
+ PartitionArrayCnt++;
2871
+
2872
+ /* Partition */
2873
+ rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition);
2874
+
2875
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
2876
+ "Forgetting Fetch partition %.*s [%" PRId32
2877
+ "]",
2878
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
2879
+ rktp->rktp_partition);
2880
+ }
2881
+ if (rkt_last != NULL) {
2882
+ /* Update last topic's PartitionArrayCnt */
2883
+ rd_kafka_buf_finalize_arraycnt(
2884
+ rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
2885
+ /* Topic tags */
2886
+ rd_kafka_buf_write_tags_empty(rkbuf);
2887
+ }
2888
+ /* Update TopicArrayCnt */
2889
+ rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt,
2890
+ TopicArrayCnt);
2891
+ } else {
2892
+ /* ForgottenToppars */
2893
+ rd_kafka_buf_write_arraycnt(rkbuf, 0);
2894
+ }
2895
+
2896
+ /* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */
2897
+ if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000)
2898
+ rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING;
2899
+
2900
+ /* Use configured timeout */
2901
+ rd_kafka_buf_set_timeout(rkbuf,
2902
+ rkb->rkb_rk->rk_conf.socket_timeout_ms +
2903
+ rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
2904
+ now);
2905
+
2906
+ /* Copy toppars_to_add/toppars_to_forget into adding/forgetting lists
2907
+ * just before sending. On response, session_update() will move
2908
+ * adding_toppars into toppars_in_session and remove from
2909
+ * toppars_to_add. */
2910
+ if (rkb->rkb_share_fetch_session.toppars_to_add)
2911
+ rkb->rkb_share_fetch_session.adding_toppars =
2912
+ rd_list_copy(rkb->rkb_share_fetch_session.toppars_to_add,
2913
+ rd_kafka_toppar_list_copy, NULL);
2914
+ if (rkb->rkb_share_fetch_session.toppars_to_forget)
2915
+ rkb->rkb_share_fetch_session.forgetting_toppars =
2916
+ rd_list_copy(rkb->rkb_share_fetch_session.toppars_to_forget,
2917
+ rd_kafka_toppar_list_copy, NULL);
2918
+
2919
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
2920
+ "Issuing ShareFetch request (max wait %dms, min %d bytes, "
2921
+ "max %d bytes, max %d records) with %d toppars",
2922
+ wait_max_ms, min_bytes, max_bytes, max_records, cnt);
2923
+ rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_share_fetch_reply,
2924
+ rko_orig);
2925
+
2926
+ return;
2927
+ }
2928
+
2929
+ /**
2930
+ * @brief Build and send a ShareAcknowledge request.
2931
+ *
2932
+ * Used for ack-only requests (no fetching) and session close (epoch=-1).
2933
+ * ShareAcknowledge carries GroupId, MemberId, ShareSessionEpoch, and
2934
+ * acknowledgement batches only — no fetch parameters or forgotten topics.
2935
+ *
2936
+ * @param rkb Broker to send request to.
2937
+ * @param group_id Consumer group id.
2938
+ * @param member_id Consumer member id.
2939
+ * @param share_session_epoch Session epoch (-1 for close).
2940
+ * @param rko_orig The originating SHARE_FETCH op (carries ack_details).
2941
+ * @param now Current timestamp.
2942
+ *
2943
+ * @locality broker thread
2944
+ */
2945
+ void rd_kafka_ShareAcknowledgeRequest(rd_kafka_broker_t *rkb,
2946
+ const rd_kafkap_str_t *group_id,
2947
+ const rd_kafkap_str_t *member_id,
2948
+ int32_t share_session_epoch,
2949
+ rd_kafka_op_t *rko_orig,
2950
+ rd_ts_t now) {
2951
+ rd_kafka_buf_t *rkbuf;
2952
+ int cnt = 0;
2953
+ size_t of_TopicArrayCnt = 0;
2954
+ int TopicArrayCnt = 0;
2955
+ size_t of_PartitionArrayCnt = 0;
2956
+ int PartitionArrayCnt = 0;
2957
+ size_t rkbuf_size = 0;
2958
+ rd_list_t *ack_details =
2959
+ rko_orig ? rko_orig->rko_u.share_fetch.ack_details : NULL;
2960
+ int ack_details_cnt = ack_details ? rd_list_cnt(ack_details) : 0;
2961
+ int total_ack_entries = 0;
2962
+ int64_t total_ack_records = 0;
2963
+ /* FirstOffset + LastOffset + AcknowledgementType per ack entry */
2964
+ size_t acknowledgement_size = 8 + 8 + 1;
2965
+
2966
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
2967
+ "ack_details_cnt=%d, share_session_epoch=%" PRId32,
2968
+ ack_details_cnt, share_session_epoch);
2969
+
2970
+ /* Calculate buffer size */
2971
+ if (group_id)
2972
+ rkbuf_size += RD_KAFKAP_STR_SIZE(group_id);
2973
+ if (member_id)
2974
+ rkbuf_size += RD_KAFKAP_STR_SIZE(member_id);
2975
+ /* ShareSessionEpoch + TopicArrayCnt */
2976
+ rkbuf_size += 4 + 4;
2977
+ /* N x (topic id + partition id) for ack details */
2978
+ rkbuf_size += (ack_details_cnt * (32 + 4));
2979
+ /* Sum total ack entries and total records across all
2980
+ * ack_details batches. */
2981
+ rd_kafka_share_ack_details_totals(ack_details, &total_ack_entries,
2982
+ &total_ack_records);
2983
+ /* Accumulate standalone record-level acknowledgements for
2984
+ * consumer.share.fetch.manager.acknowledgements.send.{total,rate}. */
2985
+ if (total_ack_records > 0)
2986
+ rd_atomic64_add(
2987
+ &rkb->rkb_rk->rk_telemetry.acknowledgements_send_total,
2988
+ total_ack_records);
2989
+
2990
+ rkbuf_size += (total_ack_entries * acknowledgement_size);
2991
+
2992
+ rkbuf = rd_kafka_buf_new_flexver_request(
2993
+ rkb, RD_KAFKAP_ShareAcknowledge, 1, rkbuf_size, rd_true);
2994
+
2995
+ rd_kafka_buf_ApiVersion_set(
2996
+ rkbuf,
2997
+ rd_kafka_broker_ApiVersion_supported(
2998
+ rkb, RD_KAFKAP_ShareAcknowledge, 1, 1, NULL),
2999
+ 0);
3000
+
3001
+ /* GroupId */
3002
+ rd_kafka_buf_write_kstr(rkbuf, group_id);
3003
+
3004
+ /* MemberId */
3005
+ rd_kafka_buf_write_kstr(rkbuf, member_id);
3006
+
3007
+ /* ShareSessionEpoch */
3008
+ rd_kafka_buf_write_i32(rkbuf, share_session_epoch);
3009
+
3010
+ /* Topics array with acknowledgement batches */
3011
+ of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
3012
+
3013
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3014
+ "Building ShareAcknowledge request with %d ack toppars "
3015
+ "and %d total ack entries",
3016
+ ack_details_cnt, total_ack_entries);
3017
+
3018
+ if (ack_details) {
3019
+ rd_kafka_Uuid_t *topic_id_last = NULL;
3020
+ rd_kafka_share_ack_batches_t *batches;
3021
+ rd_kafka_topic_partition_private_t *parpriv;
3022
+ rd_kafka_share_ack_batch_entry_t *entry;
3023
+ int k, m, entries_cnt;
3024
+
3025
+ RD_LIST_FOREACH(batches, ack_details, k) {
3026
+ parpriv = (rd_kafka_topic_partition_private_t *)
3027
+ batches->rktpar->_private;
3028
+ rd_dassert(parpriv != NULL);
3029
+
3030
+ if (topic_id_last == NULL ||
3031
+ rd_kafka_Uuid_cmp(*topic_id_last,
3032
+ parpriv->topic_id) != 0) {
3033
+ if (topic_id_last != NULL) {
3034
+ rd_kafka_buf_finalize_arraycnt(
3035
+ rkbuf, of_PartitionArrayCnt,
3036
+ PartitionArrayCnt);
3037
+ rd_kafka_buf_write_tags_empty(rkbuf);
3038
+ }
3039
+
3040
+ /* Topic ID */
3041
+ rd_kafka_buf_write_uuid(rkbuf,
3042
+ &parpriv->topic_id);
3043
+
3044
+ TopicArrayCnt++;
3045
+ topic_id_last = &parpriv->topic_id;
3046
+ of_PartitionArrayCnt =
3047
+ rd_kafka_buf_write_arraycnt_pos(rkbuf);
3048
+ PartitionArrayCnt = 0;
3049
+ }
3050
+
3051
+ PartitionArrayCnt++;
3052
+
3053
+ /* Partition */
3054
+ rd_kafka_buf_write_i32(rkbuf,
3055
+ batches->rktpar->partition);
3056
+
3057
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3058
+ "Adding ack for topic %s [%" PRId32
3059
+ "] with %d entries",
3060
+ batches->rktpar->topic,
3061
+ batches->rktpar->partition,
3062
+ rd_list_cnt(&batches->entries));
3063
+
3064
+ /* Write acknowledgement batches */
3065
+ entries_cnt = rd_list_cnt(&batches->entries);
3066
+ rd_kafka_buf_write_arraycnt(rkbuf, entries_cnt);
3067
+
3068
+ RD_LIST_FOREACH(entry, &batches->entries, m) {
3069
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3070
+ "Adding ack entry with start offset "
3071
+ "%" PRId64 ", end offset %" PRId64
3072
+ ", type %d",
3073
+ entry->start_offset,
3074
+ entry->end_offset, entry->types[0]);
3075
+ /* FirstOffset */
3076
+ rd_kafka_buf_write_i64(rkbuf,
3077
+ entry->start_offset);
3078
+ /* LastOffset */
3079
+ rd_kafka_buf_write_i64(rkbuf,
3080
+ entry->end_offset);
3081
+ /* AcknowledgeTypes */
3082
+ rd_kafka_buf_write_arraycnt(rkbuf, 1);
3083
+ rd_kafka_buf_write_i8(rkbuf,
3084
+ (int8_t)entry->types[0]);
3085
+ /* Acknowledgement tags */
3086
+ rd_kafka_buf_write_tags_empty(rkbuf);
3087
+ }
3088
+
3089
+ /* Partition tags */
3090
+ rd_kafka_buf_write_tags_empty(rkbuf);
3091
+
3092
+ cnt++;
3093
+ }
3094
+
3095
+ /* Finalize last topic from ack_details */
3096
+ if (topic_id_last != NULL) {
3097
+ rd_kafka_buf_finalize_arraycnt(
3098
+ rkbuf, of_PartitionArrayCnt, PartitionArrayCnt);
3099
+ rd_kafka_buf_write_tags_empty(rkbuf);
3100
+ }
3101
+ }
3102
+
3103
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3104
+ "ShareAcknowledge Request with %d toppars on %d topics", cnt,
3105
+ TopicArrayCnt);
3106
+
3107
+ /* Update TopicArrayCnt */
3108
+ rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
3109
+
3110
+ /* Wire RPC timeout is socket.timeout.ms, decoupled from any
3111
+ * caller-side commit_sync deadline. Interactions covered by
3112
+ * tests/0182-share_consumer_error_handling_mock.c
3113
+ * (do_test_socket_timeout_full_ack_then_more). */
3114
+ rd_kafka_buf_set_timeout(rkbuf, rkb->rkb_rk->rk_conf.socket_timeout_ms,
3115
+ now);
3116
+
3117
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3118
+ "Issuing ShareAcknowledge request with %d toppars", cnt);
3119
+ rd_kafka_broker_buf_enq1(
3120
+ rkb, rkbuf, rd_kafka_broker_share_acknowledge_reply, rko_orig);
3121
+ }
3122
+
3123
+
3124
+ void rd_kafka_broker_share_fetch_session_clear(rd_kafka_broker_t *rkb) {
3125
+ /* Clear toppars in session */
3126
+ rd_rkb_dbg(
3127
+ rkb, BROKER, "SHARESESSION",
3128
+ "Clearing %d toppars from ShareFetch session",
3129
+ rd_list_cnt(rkb->rkb_share_fetch_session.toppars_in_session));
3130
+ rd_list_clear(rkb->rkb_share_fetch_session.toppars_in_session);
3131
+
3132
+ /* Clear toppars to add */
3133
+ if (rkb->rkb_share_fetch_session.toppars_to_add) {
3134
+ rd_rkb_dbg(
3135
+ rkb, BROKER, "SHARESESSION",
3136
+ "Clearing %d toppars to add from ShareFetch session on "
3137
+ "clear",
3138
+ rd_list_cnt(rkb->rkb_share_fetch_session.toppars_to_add));
3139
+ rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_add);
3140
+ rkb->rkb_share_fetch_session.toppars_to_add = NULL;
3141
+ }
3142
+
3143
+ /* Clear toppars to forget */
3144
+ if (rkb->rkb_share_fetch_session.toppars_to_forget) {
3145
+ rd_rkb_dbg(rkb, BROKER, "SHARESESSION",
3146
+ "Clearing %d toppars to forget from ShareFetch "
3147
+ "session on clear",
3148
+ rd_list_cnt(
3149
+ rkb->rkb_share_fetch_session.toppars_to_forget));
3150
+ rd_list_destroy(rkb->rkb_share_fetch_session.toppars_to_forget);
3151
+ rkb->rkb_share_fetch_session.toppars_to_forget = NULL;
3152
+ }
3153
+
3154
+ /* Clear adding toppars */
3155
+ if (rkb->rkb_share_fetch_session.adding_toppars) {
3156
+ rd_rkb_dbg(
3157
+ rkb, BROKER, "SHARESESSION",
3158
+ "Clearing %d adding toppars from ShareFetch session on "
3159
+ "clear",
3160
+ rd_list_cnt(rkb->rkb_share_fetch_session.adding_toppars));
3161
+ rd_list_destroy(rkb->rkb_share_fetch_session.adding_toppars);
3162
+ rkb->rkb_share_fetch_session.adding_toppars = NULL;
3163
+ }
3164
+
3165
+ /* Clear forgetting toppars */
3166
+ if (rkb->rkb_share_fetch_session.forgetting_toppars) {
3167
+ rd_rkb_dbg(
3168
+ rkb, BROKER, "SHARESESSION",
3169
+ "Clearing %d forgetting toppars from ShareFetch session on "
3170
+ "clear",
3171
+ rd_list_cnt(
3172
+ rkb->rkb_share_fetch_session.forgetting_toppars));
3173
+ rd_list_destroy(
3174
+ rkb->rkb_share_fetch_session.forgetting_toppars);
3175
+ rkb->rkb_share_fetch_session.forgetting_toppars = NULL;
3176
+ }
3177
+
3178
+ /*
3179
+ * This allows us to avoid future changes to the closed share session
3180
+ * Toppar add/remove functions do not allow updates if epoch is -1
3181
+ * Avoid future changes to the closed share session */
3182
+ rkb->rkb_share_fetch_session.epoch = -1;
3183
+ }
3184
+
3185
+ void rd_kafka_broker_share_fetch_session_leave(rd_kafka_broker_t *rkb,
3186
+ rd_kafka_op_t *rko_orig,
3187
+ rd_ts_t now) {
3188
+ rd_kafka_cgrp_t *rkcg = rkb->rkb_rk->rk_cgrp;
3189
+
3190
+ if (rkb->rkb_share_fetch_session.epoch > 0) {
3191
+ rd_rkb_dbg(rkb, BROKER, "SHARESESSION",
3192
+ "Processing SHARE_FETCH op: "
3193
+ "should_leave = true");
3194
+
3195
+ /* Set epoch to -1 to signal session close before sending
3196
+ * request. This ensures session_update_epoch() skips
3197
+ * incrementing on reply. */
3198
+ rkb->rkb_share_fetch_session.epoch = -1;
3199
+ rd_kafka_ShareAcknowledgeRequest(
3200
+ rkb, rkcg->rkcg_group_id, rkcg->rkcg_member_id,
3201
+ -1, /* epoch=-1 signals session close */
3202
+ rko_orig, now);
3203
+ } else {
3204
+ rd_rkb_dbg(rkb, BROKER, "SHARESESSION",
3205
+ "Ignoring SHARE_FETCH op with "
3206
+ "should_leave = true: "
3207
+ "no active session");
3208
+
3209
+ if (rkb->rkb_share_fetch_session.epoch == 0)
3210
+ /* Required as it is possible that we were about
3211
+ * to establish a session */
3212
+ rd_kafka_broker_share_fetch_session_clear(rkb);
3213
+ rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
3214
+ rko_orig, RD_KAFKA_RESP_ERR_SHARE_SESSION_NOT_FOUND);
3215
+ }
3216
+ }
3217
+
3218
+ void rd_kafka_broker_share_rpc(rd_kafka_broker_t *rkb,
3219
+ rd_kafka_op_t *rko_orig,
3220
+ rd_ts_t now) {
3221
+
3222
+ rd_kafka_cgrp_t *rkcg = rkb->rkb_rk->rk_cgrp;
3223
+ int32_t max_records = 0;
3224
+ rd_list_t *ack_details = rko_orig->rko_u.share_fetch.ack_details;
3225
+ rd_bool_t has_ack_details = ack_details && rd_list_cnt(ack_details) > 0;
3226
+ rd_list_t *toppars_to_add = NULL;
3227
+ rd_list_t *toppars_to_forget = NULL;
3228
+ rd_list_t *stripped_acks = NULL;
3229
+
3230
+ if (!rko_orig->rko_u.share_fetch.should_fetch && !has_ack_details) {
3231
+ rd_rkb_dbg(rkb, FETCH, "SHARERPC",
3232
+ "Not sending Share RPC: "
3233
+ "no fetch requested and no acknowledgements");
3234
+ rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
3235
+ rko_orig, RD_KAFKA_RESP_ERR__NOOP);
3236
+ return;
3237
+ }
3238
+
3239
+ if (!rkcg->rkcg_member_id) {
3240
+ rd_rkb_dbg(rkb, FETCH, "SHARERPC",
3241
+ "Share RPC requested without member_id");
3242
+ rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
3243
+ rko_orig, RD_KAFKA_RESP_ERR__INVALID_ARG);
3244
+ return;
3245
+ }
3246
+
3247
+ if (!rko_orig->rko_u.share_fetch.should_fetch) {
3248
+ /* Ack-only: use ShareAcknowledge RPC.
3249
+ *
3250
+ * If session epoch is 0 (new consumer or post-reset),
3251
+ * the broker has no session state to acknowledge
3252
+ * against. Fail the acks locally with
3253
+ * INVALID_SHARE_SESSION_EPOCH instead of sending,
3254
+ * since the broker would reject the request. */
3255
+ if (rkb->rkb_share_fetch_session.epoch == 0) {
3256
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3257
+ "Failing %d ack batches locally: "
3258
+ "session epoch is 0 (no session)",
3259
+ rd_list_cnt(ack_details));
3260
+ rd_kafka_share_fetch_op_reply_and_update_ack_details_with_err(
3261
+ rko_orig,
3262
+ RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH);
3263
+ return;
3264
+ }
3265
+
3266
+ /* TODO KIP-932: add a defensive per-batch leader-stale
3267
+ * strip here (mirroring the segregation-time check) to
3268
+ * cover the window where the cached leader changes
3269
+ * between segregation on the main thread and this RPC
3270
+ * send on the broker thread. It must run AFTER the
3271
+ * epoch==0 check above so the local-only failure path
3272
+ * stays identical to the broker-roundtrip flow. */
3273
+
3274
+ rd_rkb_dbg(rkb, FETCH, "SHAREACK",
3275
+ "Sending ShareAcknowledge Request with"
3276
+ " acknowledgements");
3277
+
3278
+ rd_kafka_ShareAcknowledgeRequest(
3279
+ rkb, rkcg->rkcg_group_id, rkcg->rkcg_member_id,
3280
+ rkb->rkb_share_fetch_session.epoch, rko_orig, now);
3281
+ return;
3282
+ }
3283
+
3284
+ max_records = rkb->rkb_rk->rk_conf.share.max_poll_records;
3285
+ toppars_to_add = rkb->rkb_share_fetch_session.toppars_to_add;
3286
+ toppars_to_forget = rkb->rkb_share_fetch_session.toppars_to_forget;
3287
+
3288
+ /* If session epoch is 0 (new/reset session) and we have
3289
+ * piggybacked acks, the broker has no session state to ack
3290
+ * against. Fail the acks locally with
3291
+ * INVALID_SHARE_SESSION_EPOCH and strip them from the wire
3292
+ * request. The ShareFetch itself still goes out to establish
3293
+ * the session. */
3294
+ if (rkb->rkb_share_fetch_session.epoch == 0 && has_ack_details) {
3295
+ rd_kafka_share_ack_batches_t *batch;
3296
+ int i;
3297
+
3298
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
3299
+ "Stripping %d piggybacked ack batches: "
3300
+ "session epoch is 0 (no session)",
3301
+ rd_list_cnt(ack_details));
3302
+
3303
+ /* Pre-set each batch's err. Conditional buf-cb init
3304
+ * preserves this; main reply handler propagates to
3305
+ * commit_sync results / acknowledgement callback. */
3306
+ RD_LIST_FOREACH(batch, ack_details, i) {
3307
+ batch->rktpar->err =
3308
+ RD_KAFKA_RESP_ERR_INVALID_SHARE_SESSION_EPOCH;
3309
+ }
3310
+
3311
+ /* Detach so ShareFetchRequest builds without the ack
3312
+ * data section. Restored after the request is built
3313
+ * so the buf reply handler / main thread reply
3314
+ * handler can still read the per-batch err. */
3315
+ stripped_acks = rko_orig->rko_u.share_fetch.ack_details;
3316
+ rko_orig->rko_u.share_fetch.ack_details = NULL;
3317
+ has_ack_details = rd_false;
3318
+ }
3319
+
3320
+ /* TODO KIP-932: add a defensive per-batch leader-stale strip
3321
+ * here (mirroring the segregation-time check) to cover the
3322
+ * window where the cached leader changes between segregation
3323
+ * on the main thread and this RPC send on the broker thread.
3324
+ * It must run AFTER the epoch==0 strip above so the local-only
3325
+ * failure path stays identical to the broker-roundtrip flow. */
3326
+
3327
+ rd_rkb_dbg(rkb, FETCH, "SHAREFETCH",
3328
+ "Sending ShareFetch Request (epoch %" PRId32
3329
+ ") with%s%s%s fetching messages",
3330
+ rkb->rkb_share_fetch_session.epoch,
3331
+ has_ack_details ? " acknowledgements," : "",
3332
+ toppars_to_add ? " new topics," : "",
3333
+ toppars_to_forget ? " forgotten toppars," : "");
3334
+
3335
+ rd_kafka_ShareFetchRequest(
3336
+ rkb, rkcg->rkcg_group_id, /* group_id */
3337
+ rkcg->rkcg_member_id, /* member_id */
3338
+ rkb->rkb_share_fetch_session.epoch, /* share_session_epoch */
3339
+ rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
3340
+ rkb->rkb_rk->rk_conf.fetch_min_bytes,
3341
+ rkb->rkb_rk->rk_conf.fetch_max_bytes, max_records,
3342
+ max_records, /* batch_size: same value as max_records,
3343
+ * both sourced from max.poll.records */
3344
+ toppars_to_add, /* toppars to add to session */
3345
+ toppars_to_forget, /* forgetting toppars */
3346
+ rko_orig, /* rko (carries ack_details) */
3347
+ now);
3348
+
3349
+ /* Restore ack_details so the buf reply handler / main thread
3350
+ * reply handler can read the pre-set per-batch err. */
3351
+ if (stripped_acks)
3352
+ rko_orig->rko_u.share_fetch.ack_details = stripped_acks;
3353
+ }
3354
+
3355
+ /**
3356
+ * @brief Build and send a Fetch request message for all underflowed toppars
3357
+ * for a specific broker.
3358
+ *
3359
+ * @returns the number of partitions included in the FetchRequest, if any.
3360
+ *
3361
+ * @locality broker thread
3362
+ */
3363
+ int rd_kafka_broker_fetch_toppars(rd_kafka_broker_t *rkb, rd_ts_t now) {
3364
+ rd_kafka_toppar_t *rktp;
3365
+ rd_kafka_buf_t *rkbuf;
3366
+ int cnt = 0;
3367
+ size_t of_TopicArrayCnt = 0;
3368
+ int TopicArrayCnt = 0;
3369
+ size_t of_PartitionArrayCnt = 0;
3370
+ int PartitionArrayCnt = 0;
3371
+ rd_kafka_topic_t *rkt_last = NULL;
3372
+ int16_t ApiVersion = 0;
3373
+
3374
+ /* Create buffer and segments:
3375
+ * 1 x ReplicaId MaxWaitTime MinBytes TopicArrayCnt
3376
+ * N x topic name
3377
+ * N x PartitionArrayCnt Partition FetchOffset MaxBytes
3378
+ * where N = number of toppars.
3379
+ * Since we dont keep track of the number of topics served by
3380
+ * this broker, only the partition count, we do a worst-case calc
3381
+ * when allocating and assume each partition is on its own topic
3382
+ */
3383
+
3384
+ if (unlikely(rkb->rkb_active_toppar_cnt == 0))
3385
+ return 0;
3386
+
3387
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(rkb, RD_KAFKAP_Fetch,
3388
+ 0, 16, NULL);
3389
+
3390
+ /* Fallback to version 12 if topic id is null which can happen if
3391
+ * inter.broker.protocol.version is < 2.8 */
3392
+ if (ApiVersion > 12 && !can_use_topic_ids(rkb))
3393
+ ApiVersion = 12;
3394
+
3395
+ rkbuf = rd_kafka_buf_new_flexver_request(
3396
+ rkb, RD_KAFKAP_Fetch, 1,
3397
+ /* MaxWaitTime+MinBytes+MaxBytes+IsolationLevel+
3398
+ * SessionId+Epoch+TopicCnt */
3399
+ 4 + 4 + 4 + 1 + 4 + 4 + 4 +
3400
+ /* N x PartCnt+Partition+CurrentLeaderEpoch+FetchOffset+
3401
+ * LastFetchedEpoch+LogStartOffset+MaxBytes+?TopicNameLen?*/
3402
+ (rkb->rkb_active_toppar_cnt *
3403
+ (4 + 4 + 4 + 8 + 4 + 8 + 4 + 40)) +
3404
+ /* ForgottenTopicsCnt */
3405
+ 4 +
3406
+ /* N x ForgottenTopicsData */
3407
+ 0,
3408
+ ApiVersion >= 12);
3409
+
3410
+ if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2)
3411
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
3412
+ RD_KAFKA_FEATURE_MSGVER2);
3413
+ else if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER1)
3414
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
3415
+ RD_KAFKA_FEATURE_MSGVER1);
3416
+ else if (rkb->rkb_features & RD_KAFKA_FEATURE_THROTTLETIME)
3417
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion,
3418
+ RD_KAFKA_FEATURE_THROTTLETIME);
3419
+
3420
+
3421
+ /* FetchRequest header */
3422
+ if (rd_kafka_buf_ApiVersion(rkbuf) <= 14)
3423
+ /* ReplicaId */
3424
+ rd_kafka_buf_write_i32(rkbuf, -1);
3425
+
3426
+ /* MaxWaitTime */
3427
+ rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_wait_max_ms);
3428
+ /* MinBytes */
3429
+ rd_kafka_buf_write_i32(rkbuf, rkb->rkb_rk->rk_conf.fetch_min_bytes);
3430
+
3431
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 3)
3432
+ /* MaxBytes */
3433
+ rd_kafka_buf_write_i32(rkbuf,
3434
+ rkb->rkb_rk->rk_conf.fetch_max_bytes);
3435
+
3436
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 4)
3437
+ /* IsolationLevel */
3438
+ rd_kafka_buf_write_i8(rkbuf,
3439
+ rkb->rkb_rk->rk_conf.isolation_level);
3440
+
3441
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 7) {
3442
+ /* SessionId */
3443
+ rd_kafka_buf_write_i32(rkbuf, 0);
3444
+ /* Epoch */
3445
+ rd_kafka_buf_write_i32(rkbuf, -1);
3446
+ }
3447
+
3448
+ /* Write zero TopicArrayCnt but store pointer for later update */
3449
+ of_TopicArrayCnt = rd_kafka_buf_write_arraycnt_pos(rkbuf);
3450
+
3451
+ /* Prepare map for storing the fetch version for each partition,
3452
+ * this will later be checked in Fetch response to purge outdated
3453
+ * responses (e.g., after a seek). */
3454
+ rkbuf->rkbuf_rktp_vers =
3455
+ rd_list_new(0, (void *)rd_kafka_toppar_ver_destroy);
3456
+ rd_list_prealloc_elems(rkbuf->rkbuf_rktp_vers,
3457
+ sizeof(struct rd_kafka_toppar_ver),
3458
+ rkb->rkb_active_toppar_cnt, 0);
3459
+
3460
+ /* Round-robin start of the list. */
3461
+ rktp = rkb->rkb_active_toppar_next;
3462
+ do {
3463
+ struct rd_kafka_toppar_ver *tver;
3464
+
3465
+ if (rkt_last != rktp->rktp_rkt) {
3466
+ if (rkt_last != NULL) {
3467
+ /* Update PartitionArrayCnt */
3468
+ rd_kafka_buf_finalize_arraycnt(
3469
+ rkbuf, of_PartitionArrayCnt,
3470
+ PartitionArrayCnt);
3471
+ /* Topic tags */
3472
+ rd_kafka_buf_write_tags_empty(rkbuf);
3473
+ }
3474
+
3475
+ /* TODO: This is not thread safe as topic can
3476
+ be recreated in which case topic id is
3477
+ updated from the main thread and we are
3478
+ sending topic id from broker thread.*/
3479
+ if (rd_kafka_buf_ApiVersion(rkbuf) > 12) {
3480
+ /* Topic id must be non-zero here */
3481
+ rd_dassert(!RD_KAFKA_UUID_IS_ZERO(
3482
+ rktp->rktp_rkt->rkt_topic_id));
3483
+ /* Topic ID */
3484
+ rd_kafka_buf_write_uuid(
3485
+ rkbuf, &rktp->rktp_rkt->rkt_topic_id);
3486
+ } else {
3487
+ /* Topic name */
3488
+ rd_kafka_buf_write_kstr(
3489
+ rkbuf, rktp->rktp_rkt->rkt_topic);
3490
+ }
3491
+
3492
+ TopicArrayCnt++;
3493
+ rkt_last = rktp->rktp_rkt;
3494
+ /* Partition count */
3495
+ of_PartitionArrayCnt =
3496
+ rd_kafka_buf_write_arraycnt_pos(rkbuf);
3497
+ PartitionArrayCnt = 0;
3498
+ }
3499
+
3500
+ PartitionArrayCnt++;
3501
+
3502
+ /* Partition */
3503
+ rd_kafka_buf_write_i32(rkbuf, rktp->rktp_partition);
3504
+
3505
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 9) {
3506
+ /* CurrentLeaderEpoch */
3507
+ if (rktp->rktp_leader_epoch < 0 &&
3508
+ rd_kafka_has_reliable_leader_epochs(rkb)) {
3509
+ /* If current leader epoch is set to -1 and
3510
+ * the broker has reliable leader epochs,
3511
+ * send 0 instead, so that epoch is checked
3512
+ * and optionally metadata is refreshed.
3513
+ * This can happen if metadata is read initially
3514
+ * without an existing topic (see
3515
+ * rd_kafka_topic_metadata_update2).
3516
+ */
3517
+ rd_kafka_buf_write_i32(rkbuf, 0);
3518
+ } else {
3519
+ rd_kafka_buf_write_i32(rkbuf,
3520
+ rktp->rktp_leader_epoch);
3521
+ }
3522
+ }
3523
+ /* FetchOffset */
3524
+ rd_kafka_buf_write_i64(rkbuf,
3525
+ rktp->rktp_offsets.fetch_pos.offset);
3526
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 12)
3527
+ /* LastFetchedEpoch - only used by follower replica */
3528
+ rd_kafka_buf_write_i32(rkbuf, -1);
3529
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 5)
3530
+ /* LogStartOffset - only used by follower replica */
3531
+ rd_kafka_buf_write_i64(rkbuf, -1);
3532
+
3533
+ /* MaxBytes */
3534
+ rd_kafka_buf_write_i32(rkbuf, rktp->rktp_fetch_msg_max_bytes);
3535
+
3536
+ /* Partition tags */
3537
+ rd_kafka_buf_write_tags_empty(rkbuf);
3538
+
3539
+ rd_rkb_dbg(rkb, FETCH, "FETCH",
3540
+ "Fetch topic %.*s [%" PRId32 "] at offset %" PRId64
3541
+ " (leader epoch %" PRId32
3542
+ ", current leader epoch %" PRId32 ", v%d)",
3543
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
3544
+ rktp->rktp_partition,
3545
+ rktp->rktp_offsets.fetch_pos.offset,
3546
+ rktp->rktp_offsets.fetch_pos.leader_epoch,
3547
+ rktp->rktp_leader_epoch, rktp->rktp_fetch_version);
3548
+
3549
+ /* We must have a valid fetch offset when we get here */
3550
+ rd_dassert(rktp->rktp_offsets.fetch_pos.offset >= 0);
3551
+
3552
+ /* Add toppar + op version mapping. */
3553
+ tver = rd_list_add(rkbuf->rkbuf_rktp_vers, NULL);
3554
+ tver->rktp = rd_kafka_toppar_keep(rktp);
3555
+ tver->version = rktp->rktp_fetch_version;
3556
+
3557
+ cnt++;
3558
+ } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp,
3559
+ rktp_activelink)) !=
3560
+ rkb->rkb_active_toppar_next);
3561
+
3562
+ /* Update next toppar to fetch in round-robin list. */
3563
+ rd_kafka_broker_active_toppar_next(
3564
+ rkb, rktp ? CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars, rktp,
3565
+ rktp_activelink)
3566
+ : NULL);
3567
+
3568
+ rd_rkb_dbg(rkb, FETCH, "FETCH", "Fetch %i/%i/%i toppar(s)", cnt,
3569
+ rkb->rkb_active_toppar_cnt, rkb->rkb_toppar_cnt);
3570
+ if (!cnt) {
3571
+ rd_kafka_buf_destroy(rkbuf);
3572
+ return cnt;
3573
+ }
3574
+
3575
+ if (rkt_last != NULL) {
3576
+ /* Update last topic's PartitionArrayCnt */
3577
+ rd_kafka_buf_finalize_arraycnt(rkbuf, of_PartitionArrayCnt,
3578
+ PartitionArrayCnt);
3579
+ /* Topic tags */
3580
+ rd_kafka_buf_write_tags_empty(rkbuf);
3581
+ }
3582
+
3583
+ /* Update TopicArrayCnt */
3584
+ rd_kafka_buf_finalize_arraycnt(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
3585
+
3586
+
3587
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 7)
3588
+ /* Length of the ForgottenTopics list (KIP-227). Broker
3589
+ * use only - not used by the consumer. */
3590
+ rd_kafka_buf_write_arraycnt(rkbuf, 0);
3591
+
3592
+ if (rd_kafka_buf_ApiVersion(rkbuf) >= 11)
3593
+ /* RackId */
3594
+ rd_kafka_buf_write_kstr(rkbuf,
3595
+ rkb->rkb_rk->rk_conf.client_rack);
3596
+
3597
+ /* Consider Fetch requests blocking if fetch.wait.max.ms >= 1s */
3598
+ if (rkb->rkb_rk->rk_conf.fetch_wait_max_ms >= 1000)
3599
+ rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING;
3600
+
3601
+ /* Use configured timeout */
3602
+ rd_kafka_buf_set_timeout(rkbuf,
3603
+ rkb->rkb_rk->rk_conf.socket_timeout_ms +
3604
+ rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
3605
+ now);
3606
+
3607
+ /* Sort toppar versions for quicker lookups in Fetch response. */
3608
+ rd_list_sort(rkbuf->rkbuf_rktp_vers, rd_kafka_toppar_ver_cmp);
3609
+
3610
+ rkb->rkb_fetching = 1;
3611
+ rd_kafka_broker_buf_enq1(rkb, rkbuf, rd_kafka_broker_fetch_reply, NULL);
3612
+
3613
+ return cnt;
3614
+ }
3615
+
3616
+ /**
3617
+ * @brief Decide whether it should start fetching from next fetch start
3618
+ * or continue with current fetch pos.
3619
+ *
3620
+ * @param rktp the toppar
3621
+ *
3622
+ * @returns rd_true if it should start fetching from next fetch start,
3623
+ * rd_false otherwise.
3624
+ *
3625
+ * @locality any
3626
+ * @locks toppar_lock() MUST be held
3627
+ */
3628
+ static rd_bool_t rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(
3629
+ rd_kafka_toppar_t *rktp) {
3630
+ return rktp->rktp_op_version > rktp->rktp_fetch_version ||
3631
+ rd_kafka_fetch_pos_cmp(&rktp->rktp_next_fetch_start,
3632
+ &rktp->rktp_last_next_fetch_start) ||
3633
+ rktp->rktp_offsets.fetch_pos.offset == RD_KAFKA_OFFSET_INVALID;
3634
+ }
3635
+
3636
+ /**
3637
+ * @brief Return next fetch start position:
3638
+ * if it should start fetching from next fetch start
3639
+ * or continue with current fetch pos.
3640
+ *
3641
+ * @param rktp The toppar
3642
+ *
3643
+ * @returns Next fetch start position
3644
+ *
3645
+ * @locality any
3646
+ * @locks toppar_lock() MUST be held
3647
+ */
3648
+ rd_kafka_fetch_pos_t
3649
+ rd_kafka_toppar_fetch_decide_next_fetch_start_pos(rd_kafka_toppar_t *rktp) {
3650
+ if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp))
3651
+ return rktp->rktp_next_fetch_start;
3652
+ else
3653
+ return rktp->rktp_offsets.fetch_pos;
3654
+ }
3655
+
3656
+ /**
3657
+ * @brief Decide whether this toppar should be on the fetch list or not.
3658
+ *
3659
+ * Also:
3660
+ * - update toppar's op version (for broker thread's copy)
3661
+ * - finalize statistics (move rktp_offsets to rktp_offsets_fin)
3662
+ *
3663
+ * @returns the partition's Fetch backoff timestamp, or 0 if no backoff.
3664
+ *
3665
+ * @locality broker thread
3666
+ * @locks none
3667
+ */
3668
+ rd_ts_t rd_kafka_toppar_fetch_decide(rd_kafka_toppar_t *rktp,
3669
+ rd_kafka_broker_t *rkb,
3670
+ int force_remove) {
3671
+ int should_fetch = 1;
3672
+ const char *reason = "";
3673
+ int32_t version;
3674
+ rd_ts_t ts_backoff = 0;
3675
+ rd_bool_t lease_expired = rd_false;
3676
+
3677
+ rd_kafka_toppar_lock(rktp);
3678
+
3679
+ /* Check for preferred replica lease expiry */
3680
+ lease_expired = rktp->rktp_leader_id != rktp->rktp_broker_id &&
3681
+ rd_interval(&rktp->rktp_lease_intvl,
3682
+ 5 * 60 * 1000 * 1000 /*5 minutes*/, 0) > 0;
3683
+ if (lease_expired) {
3684
+ /* delegate_to_leader() requires no locks to be held */
3685
+ rd_kafka_toppar_unlock(rktp);
3686
+ rd_kafka_toppar_delegate_to_leader(rktp);
3687
+ rd_kafka_toppar_lock(rktp);
3688
+
3689
+ reason = "preferred replica lease expired";
3690
+ should_fetch = 0;
3691
+ goto done;
3692
+ }
3693
+
3694
+ /* Forced removal from fetch list */
3695
+ if (unlikely(force_remove)) {
3696
+ reason = "forced removal";
3697
+ should_fetch = 0;
3698
+ goto done;
3699
+ }
3700
+
3701
+ if (unlikely((rktp->rktp_flags & RD_KAFKA_TOPPAR_F_REMOVE) != 0)) {
3702
+ reason = "partition removed";
3703
+ should_fetch = 0;
3704
+ goto done;
3705
+ }
3706
+
3707
+ /* Skip toppars not in active fetch state */
3708
+ if (rktp->rktp_fetch_state != RD_KAFKA_TOPPAR_FETCH_ACTIVE) {
3709
+ reason = "not in active fetch state";
3710
+ should_fetch = 0;
3711
+ goto done;
3712
+ }
3713
+
3714
+ /* Update broker thread's fetch op version */
3715
+ version = rktp->rktp_op_version;
3716
+ if (rd_kafka_toppar_fetch_decide_start_from_next_fetch_start(rktp)) {
3717
+ /* New version barrier, something was modified from the
3718
+ * control plane. Reset and start over.
3719
+ * Alternatively only the next_offset changed but not the
3720
+ * barrier, which is the case when automatically triggering
3721
+ * offset.reset (such as on PARTITION_EOF or
3722
+ * OFFSET_OUT_OF_RANGE). */
3723
+
3724
+ rd_kafka_dbg(
3725
+ rktp->rktp_rkt->rkt_rk, TOPIC, "FETCHDEC",
3726
+ "Topic %s [%" PRId32
3727
+ "]: fetch decide: "
3728
+ "updating to version %d (was %d) at %s "
3729
+ "(was %s)",
3730
+ rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
3731
+ version, rktp->rktp_fetch_version,
3732
+ rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start),
3733
+ rd_kafka_fetch_pos2str(rktp->rktp_offsets.fetch_pos));
3734
+
3735
+ rd_kafka_offset_stats_reset(&rktp->rktp_offsets);
3736
+
3737
+ /* New start offset */
3738
+ rktp->rktp_offsets.fetch_pos = rktp->rktp_next_fetch_start;
3739
+ rktp->rktp_last_next_fetch_start = rktp->rktp_next_fetch_start;
3740
+
3741
+ rktp->rktp_fetch_version = version;
3742
+
3743
+ /* Clear last error to propagate new fetch
3744
+ * errors if encountered. */
3745
+ rktp->rktp_last_error = RD_KAFKA_RESP_ERR_NO_ERROR;
3746
+
3747
+ rd_kafka_q_purge_toppar_version(rktp->rktp_fetchq, rktp,
3748
+ version);
3749
+ }
3750
+
3751
+
3752
+ if (RD_KAFKA_TOPPAR_IS_PAUSED(rktp)) {
3753
+ should_fetch = 0;
3754
+ reason = "paused";
3755
+
3756
+ } else if (RD_KAFKA_OFFSET_IS_LOGICAL(
3757
+ rktp->rktp_next_fetch_start.offset)) {
3758
+ should_fetch = 0;
3759
+ reason = "no concrete offset";
3760
+ } else if (rktp->rktp_ts_fetch_backoff > rd_clock()) {
3761
+ reason = "fetch backed off";
3762
+ ts_backoff = rktp->rktp_ts_fetch_backoff;
3763
+ should_fetch = 0;
3764
+ } else if (rd_kafka_q_len(rktp->rktp_fetchq) >=
3765
+ rkb->rkb_rk->rk_conf.queued_min_msgs) {
3766
+ /* Skip toppars who's local message queue is already above
3767
+ * the lower threshold. */
3768
+ reason = "queued.min.messages exceeded";
3769
+ ts_backoff = rd_kafka_toppar_fetch_backoff(
3770
+ rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL);
3771
+ should_fetch = 0;
3772
+
3773
+ } else if ((int64_t)rd_kafka_q_size(rktp->rktp_fetchq) >=
3774
+ rkb->rkb_rk->rk_conf.queued_max_msg_bytes) {
3775
+ reason = "queued.max.messages.kbytes exceeded";
3776
+ ts_backoff = rd_kafka_toppar_fetch_backoff(
3777
+ rkb, rktp, RD_KAFKA_RESP_ERR__QUEUE_FULL);
3778
+ should_fetch = 0;
3779
+ }
3780
+
3781
+ done:
3782
+ /* Copy offset stats to finalized place holder. */
3783
+ rktp->rktp_offsets_fin = rktp->rktp_offsets;
3784
+
3785
+ if (rktp->rktp_fetch != should_fetch) {
3786
+ rd_rkb_dbg(
3787
+ rkb, FETCH, "FETCH",
3788
+ "Topic %s [%" PRId32
3789
+ "] in state %s at %s "
3790
+ "(%d/%d msgs, %" PRId64
3791
+ "/%d kb queued, "
3792
+ "opv %" PRId32 ") is %s%s",
3793
+ rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
3794
+ rd_kafka_fetch_states[rktp->rktp_fetch_state],
3795
+ rd_kafka_fetch_pos2str(rktp->rktp_next_fetch_start),
3796
+ rd_kafka_q_len(rktp->rktp_fetchq),
3797
+ rkb->rkb_rk->rk_conf.queued_min_msgs,
3798
+ rd_kafka_q_size(rktp->rktp_fetchq) / 1024,
3799
+ rkb->rkb_rk->rk_conf.queued_max_msg_kbytes,
3800
+ rktp->rktp_fetch_version,
3801
+ should_fetch ? "fetchable" : "not fetchable: ", reason);
3802
+
3803
+ if (should_fetch) {
3804
+ rd_dassert(rktp->rktp_fetch_version > 0);
3805
+ rd_kafka_broker_active_toppar_add(
3806
+ rkb, rktp, *reason ? reason : "fetchable");
3807
+ } else {
3808
+ rd_kafka_broker_active_toppar_del(rkb, rktp, reason);
3809
+ }
3810
+ }
3811
+
3812
+ rd_kafka_toppar_unlock(rktp);
3813
+
3814
+ /* Non-fetching partitions will have an
3815
+ * indefinate backoff, unless explicitly specified. */
3816
+ if (!should_fetch && !ts_backoff)
3817
+ ts_backoff = RD_TS_MAX;
3818
+
3819
+ return ts_backoff;
3820
+ }