rdkafka 0.22.2 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +63 -3
  3. data/Gemfile +8 -0
  4. data/Gemfile.lint +14 -0
  5. data/Gemfile.lint.lock +123 -0
  6. data/README.md +19 -14
  7. data/Rakefile +21 -21
  8. data/bin/verify_kafka_warnings +39 -0
  9. data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.0.tar.gz} +0 -0
  10. data/docker-compose-ssl.yml +35 -0
  11. data/docker-compose.yml +2 -2
  12. data/ext/Rakefile +27 -27
  13. data/lib/rdkafka/abstract_handle.rb +23 -5
  14. data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
  15. data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
  16. data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
  17. data/lib/rdkafka/admin/create_acl_report.rb +3 -2
  18. data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
  19. data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
  20. data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
  21. data/lib/rdkafka/admin/create_topic_report.rb +3 -0
  22. data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
  23. data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
  24. data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
  25. data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
  26. data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
  27. data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
  28. data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
  29. data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
  30. data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
  31. data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
  32. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
  33. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
  34. data/lib/rdkafka/admin/list_offsets_handle.rb +36 -0
  35. data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
  36. data/lib/rdkafka/admin.rb +301 -135
  37. data/lib/rdkafka/bindings.rb +199 -110
  38. data/lib/rdkafka/callbacks.rb +124 -21
  39. data/lib/rdkafka/config.rb +81 -33
  40. data/lib/rdkafka/consumer/headers.rb +3 -2
  41. data/lib/rdkafka/consumer/message.rb +12 -11
  42. data/lib/rdkafka/consumer/partition.rb +8 -4
  43. data/lib/rdkafka/consumer/topic_partition_list.rb +21 -17
  44. data/lib/rdkafka/consumer.rb +397 -45
  45. data/lib/rdkafka/defaults.rb +106 -0
  46. data/lib/rdkafka/error.rb +40 -14
  47. data/lib/rdkafka/helpers/oauth.rb +45 -13
  48. data/lib/rdkafka/helpers/time.rb +5 -0
  49. data/lib/rdkafka/metadata.rb +45 -21
  50. data/lib/rdkafka/native_kafka.rb +89 -4
  51. data/lib/rdkafka/producer/delivery_handle.rb +5 -5
  52. data/lib/rdkafka/producer/delivery_report.rb +10 -6
  53. data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
  54. data/lib/rdkafka/producer.rb +168 -82
  55. data/lib/rdkafka/version.rb +6 -3
  56. data/lib/rdkafka.rb +3 -0
  57. data/package-lock.json +331 -0
  58. data/package.json +9 -0
  59. data/rdkafka.gemspec +57 -36
  60. data/renovate.json +29 -24
  61. metadata +29 -124
  62. data/.github/CODEOWNERS +0 -3
  63. data/.github/FUNDING.yml +0 -1
  64. data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
  65. data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
  66. data/.github/workflows/ci_macos_arm64.yml +0 -284
  67. data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
  68. data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
  69. data/.github/workflows/push_macos_arm64.yml +0 -54
  70. data/.github/workflows/push_ruby.yml +0 -37
  71. data/.github/workflows/verify-action-pins.yml +0 -16
  72. data/.gitignore +0 -14
  73. data/.rspec +0 -2
  74. data/.ruby-gemset +0 -1
  75. data/.ruby-version +0 -1
  76. data/.yardopts +0 -2
  77. data/ext/README.md +0 -19
  78. data/ext/build_common.sh +0 -361
  79. data/ext/build_linux_x86_64_gnu.sh +0 -306
  80. data/ext/build_linux_x86_64_musl.sh +0 -763
  81. data/ext/build_macos_arm64.sh +0 -550
  82. data/spec/rdkafka/abstract_handle_spec.rb +0 -117
  83. data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
  84. data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
  85. data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
  86. data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
  87. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
  88. data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
  89. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
  90. data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
  91. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
  92. data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
  93. data/spec/rdkafka/admin_spec.rb +0 -971
  94. data/spec/rdkafka/bindings_spec.rb +0 -199
  95. data/spec/rdkafka/callbacks_spec.rb +0 -20
  96. data/spec/rdkafka/config_spec.rb +0 -258
  97. data/spec/rdkafka/consumer/headers_spec.rb +0 -73
  98. data/spec/rdkafka/consumer/message_spec.rb +0 -139
  99. data/spec/rdkafka/consumer/partition_spec.rb +0 -57
  100. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
  101. data/spec/rdkafka/consumer_spec.rb +0 -1274
  102. data/spec/rdkafka/error_spec.rb +0 -89
  103. data/spec/rdkafka/metadata_spec.rb +0 -79
  104. data/spec/rdkafka/native_kafka_spec.rb +0 -130
  105. data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
  106. data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
  107. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
  108. data/spec/rdkafka/producer_spec.rb +0 -1345
  109. data/spec/spec_helper.rb +0 -195
@@ -2,18 +2,61 @@
2
2
 
3
3
  module Rdkafka
4
4
  # @private
5
+ #
6
+ # @note
7
+ # There are two types of responses related to errors:
8
+ # - rd_kafka_error_t - a C object that we need to remap into an error or null when no error
9
+ # - rd_kafka_resp_err_t - response error code (numeric) that we can use directly
10
+ #
11
+ # It is critical to ensure, that we handle them correctly. The result type should be:
12
+ # - rd_kafka_error_t - :pointer
13
+ # - rd_kafka_resp_err_t - :int
5
14
  module Bindings
6
15
  extend FFI::Library
7
16
 
17
+ # Returns the library extension based on the host OS
18
+ # @return [String] 'dylib' on macOS, 'so' on other systems
8
19
  def self.lib_extension
9
- if RbConfig::CONFIG['host_os'] =~ /darwin/
10
- 'dylib'
20
+ if /darwin/.match?(RbConfig::CONFIG["host_os"])
21
+ "dylib"
11
22
  else
12
- 'so'
23
+ "so"
13
24
  end
14
25
  end
15
26
 
16
- ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
27
+ # Wrap ffi_lib to provide better error messages for glibc compatibility issues
28
+ begin
29
+ ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
30
+ rescue LoadError => e
31
+ error_message = e.message
32
+
33
+ # Check if this is a glibc version mismatch error
34
+ if /GLIBC_[\d.]+['"` ]?\s*not found/i.match?(error_message)
35
+ glibc_version = error_message[/GLIBC_([\d.]+)/, 1] || "unknown"
36
+
37
+ raise Rdkafka::LibraryLoadError, <<~ERROR_MSG.strip
38
+ Failed to load librdkafka due to glibc compatibility issue.
39
+
40
+ The precompiled librdkafka binary requires glibc version #{glibc_version} or higher,
41
+ but your system has an older version installed.
42
+
43
+ To resolve this issue, you have two options:
44
+
45
+ 1. Upgrade your system to a supported platform (recommended)
46
+
47
+ 2. Force compilation from source by reinstalling without the precompiled binary:
48
+ gem install rdkafka --platform=ruby
49
+
50
+ Or if using Bundler, add to your Gemfile:
51
+ gem 'rdkafka', force_ruby_platform: true
52
+
53
+ Original error: #{error_message}
54
+ ERROR_MSG
55
+ else
56
+ # Re-raise the original error if it's not a glibc issue
57
+ raise
58
+ end
59
+ end
17
60
 
18
61
  RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
19
62
  RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
@@ -21,13 +64,17 @@ module Rdkafka
21
64
  RD_KAFKA_RESP_ERR__NOENT = -156
22
65
  RD_KAFKA_RESP_ERR_NO_ERROR = 0
23
66
 
24
- RD_KAFKA_OFFSET_END = -1
67
+ RD_KAFKA_OFFSET_END = -1
25
68
  RD_KAFKA_OFFSET_BEGINNING = -2
26
- RD_KAFKA_OFFSET_STORED = -1000
27
- RD_KAFKA_OFFSET_INVALID = -1001
69
+ RD_KAFKA_OFFSET_STORED = -1000
70
+ RD_KAFKA_OFFSET_INVALID = -1001
71
+
72
+ RD_KAFKA_PARTITION_UA = -1
73
+ RD_KAFKA_PARTITION_UA_STR = RD_KAFKA_PARTITION_UA.to_s.freeze
28
74
 
29
75
  EMPTY_HASH = {}.freeze
30
76
 
77
+ # FFI struct for size_t pointer wrapper
31
78
  class SizePtr < FFI::Struct
32
79
  layout :value, :size_t
33
80
  end
@@ -43,6 +90,12 @@ module Rdkafka
43
90
  attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
44
91
  attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
45
92
 
93
+ # Non-blocking poll variants (do not release GVL)
94
+ # These are more efficient for poll(0) calls in fiber schedulers where GVL
95
+ # release/reacquire overhead is wasteful since we don't expect to wait.
96
+ # Uses the same underlying C function but with blocking: false to skip GVL release.
97
+ attach_function :rd_kafka_poll_nb, :rd_kafka_poll, [:pointer, :int], :int, blocking: false
98
+
46
99
  # Metadata
47
100
 
48
101
  attach_function :rd_kafka_name, [:pointer], :string
@@ -51,18 +104,17 @@ module Rdkafka
51
104
  attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
52
105
  attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
53
106
 
54
- # Message struct
55
-
107
+ # FFI struct representing a Kafka message (rd_kafka_message_t)
56
108
  class Message < FFI::Struct
57
109
  layout :err, :int,
58
- :rkt, :pointer,
59
- :partition, :int32,
60
- :payload, :pointer,
61
- :len, :size_t,
62
- :key, :pointer,
63
- :key_len, :size_t,
64
- :offset, :int64,
65
- :_private, :pointer
110
+ :rkt, :pointer,
111
+ :partition, :int32,
112
+ :payload, :pointer,
113
+ :len, :size_t,
114
+ :key, :pointer,
115
+ :key_len, :size_t,
116
+ :offset, :int64,
117
+ :_private, :pointer
66
118
  end
67
119
 
68
120
  attach_function :rd_kafka_message_destroy, [:pointer], :void
@@ -71,40 +123,38 @@ module Rdkafka
71
123
  attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
72
124
  attach_function :rd_kafka_topic_name, [:pointer], :string
73
125
 
74
- # TopicPartition ad TopicPartitionList structs
75
-
126
+ # FFI struct representing a topic partition (rd_kafka_topic_partition_t)
76
127
  class TopicPartition < FFI::Struct
77
128
  layout :topic, :string,
78
- :partition, :int32,
79
- :offset, :int64,
80
- :metadata, :pointer,
81
- :metadata_size, :size_t,
82
- :opaque, :pointer,
83
- :err, :int,
84
- :_private, :pointer
129
+ :partition, :int32,
130
+ :offset, :int64,
131
+ :metadata, :pointer,
132
+ :metadata_size, :size_t,
133
+ :opaque, :pointer,
134
+ :err, :int,
135
+ :_private, :pointer
85
136
  end
86
137
 
138
+ # FFI struct representing a topic partition list (rd_kafka_topic_partition_list_t)
87
139
  class TopicPartitionList < FFI::Struct
88
140
  layout :cnt, :int,
89
- :size, :int,
90
- :elems, :pointer
141
+ :size, :int,
142
+ :elems, :pointer
91
143
  end
92
144
 
93
145
  attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
94
- attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
146
+ attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
95
147
  attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
96
148
  attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
97
149
  attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
98
150
 
99
- # Configs management
100
- #
101
- # Structs for management of configurations
102
- # Each configuration is attached to a resource and one resource can have many configuration
103
- # details. Each resource will also have separate errors results if obtaining configuration
104
- # was not possible for any reason
151
+ # FFI struct representing a config resource (rd_kafka_ConfigResource_t)
152
+ # Structs for management of configurations. Each configuration is attached to a resource
153
+ # and one resource can have many configuration details. Each resource will also have
154
+ # separate errors results if obtaining configuration was not possible for any reason
105
155
  class ConfigResource < FFI::Struct
106
156
  layout :type, :int,
107
- :name, :string
157
+ :name, :string
108
158
  end
109
159
 
110
160
  attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
@@ -133,20 +183,45 @@ module Rdkafka
133
183
  RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
134
184
  RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
135
185
 
136
- RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
137
- RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
138
- RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
186
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
187
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
188
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
139
189
  RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
140
190
 
141
- # Errors
191
+ # List Offsets
192
+ RD_KAFKA_ADMIN_OP_LISTOFFSETS = 20
193
+ RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
194
+
195
+ # rd_kafka_IsolationLevel_t
196
+ RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED = 0
197
+ RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED = 1
198
+
199
+ # rd_kafka_OffsetSpec_t
200
+ RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP = -3
201
+ RD_KAFKA_OFFSET_SPEC_EARLIEST = -2
202
+ RD_KAFKA_OFFSET_SPEC_LATEST = -1
203
+
204
+ attach_function :rd_kafka_ListOffsets, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
205
+ attach_function :rd_kafka_event_ListOffsets_result, [:pointer], :pointer
206
+ attach_function :rd_kafka_ListOffsets_result_infos, [:pointer, :pointer], :pointer
207
+ attach_function :rd_kafka_ListOffsetsResultInfo_topic_partition, [:pointer], :pointer
208
+ attach_function :rd_kafka_ListOffsetsResultInfo_timestamp, [:pointer], :int64
209
+ attach_function :rd_kafka_AdminOptions_set_isolation_level, [:pointer, :int], :pointer
210
+ attach_function :rd_kafka_topic_partition_get_leader_epoch, [:pointer], :int32
211
+
212
+ # FFI struct for error description (rd_kafka_err_desc)
142
213
  class NativeErrorDesc < FFI::Struct
143
214
  layout :code, :int,
144
- :name, :pointer,
145
- :desc, :pointer
215
+ :name, :pointer,
216
+ :desc, :pointer
146
217
  end
147
218
 
148
219
  attach_function :rd_kafka_err2name, [:int], :string
149
220
  attach_function :rd_kafka_err2str, [:int], :string
221
+ attach_function :rd_kafka_error_is_fatal, [:pointer], :int
222
+ attach_function :rd_kafka_error_is_retriable, [:pointer], :int
223
+ attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
224
+ attach_function :rd_kafka_error_destroy, [:pointer], :void
150
225
  attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
151
226
 
152
227
  # Configuration
@@ -159,6 +234,9 @@ module Rdkafka
159
234
 
160
235
  attach_function :rd_kafka_conf_new, [], :pointer
161
236
  attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
237
+ attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
238
+ attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
239
+ attach_function :rd_kafka_conf_destroy, [:pointer], :void
162
240
  callback :log_cb, [:pointer, :int, :string, :string], :void
163
241
  attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
164
242
  attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
@@ -174,6 +252,15 @@ module Rdkafka
174
252
  # Log queue
175
253
  attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
176
254
  attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
255
+ attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
256
+
257
+ # Queue IO Event Support - for fiber scheduler integration
258
+ # Enables notifications to a custom FD when queue transitions from empty to non-empty
259
+ # @param queue rd_kafka_queue_t* - the queue to monitor
260
+ # @param fd int - file descriptor to write to (provide your own pipe/eventfd)
261
+ # @param payload const void* - data to write to fd
262
+ # @param size size_t - size of payload
263
+ attach_function :rd_kafka_queue_io_event_enable, [:pointer, :int, :pointer, :size_t], :void
177
264
  # Per topic configs
178
265
  attach_function :rd_kafka_topic_conf_new, [], :pointer
179
266
  attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
@@ -182,19 +269,19 @@ module Rdkafka
182
269
  :void, [:pointer, :int, :string, :string]
183
270
  ) do |_client_ptr, level, _level_string, line|
184
271
  severity = case level
185
- when 0, 1, 2
186
- Logger::FATAL
187
- when 3
188
- Logger::ERROR
189
- when 4
190
- Logger::WARN
191
- when 5, 6
192
- Logger::INFO
193
- when 7
194
- Logger::DEBUG
195
- else
196
- Logger::UNKNOWN
197
- end
272
+ when 0, 1, 2
273
+ Logger::FATAL
274
+ when 3
275
+ Logger::ERROR
276
+ when 4
277
+ Logger::WARN
278
+ when 5, 6
279
+ Logger::INFO
280
+ when 7
281
+ Logger::DEBUG
282
+ else
283
+ Logger::UNKNOWN
284
+ end
198
285
 
199
286
  Rdkafka::Config.ensure_log_thread
200
287
  Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
@@ -220,8 +307,8 @@ module Rdkafka
220
307
  #
221
308
  # Since this cache is shared, having few consumers and/or producers in one process will
222
309
  # automatically improve the querying times even with low refresh times.
223
- (stats['topics'] || EMPTY_HASH).each do |topic_name, details|
224
- partitions_count = details['partitions'].keys.reject { |k| k == '-1' }.size
310
+ (stats["topics"] || EMPTY_HASH).each do |topic_name, details|
311
+ partitions_count = details["partitions"].keys.count { |k| !(k == RD_KAFKA_PARTITION_UA_STR) }
225
312
 
226
313
  next unless partitions_count.positive?
227
314
 
@@ -232,14 +319,15 @@ module Rdkafka
232
319
  end
233
320
 
234
321
  # Return 0 so librdkafka frees the json string
235
- 0
322
+ RD_KAFKA_RESP_ERR_NO_ERROR
236
323
  end
237
324
 
238
325
  ErrorCallback = FFI::Function.new(
239
326
  :void, [:pointer, :int, :string, :pointer]
240
- ) do |_client_prr, err_code, reason, _opaque|
327
+ ) do |client_ptr, err_code, reason, _opaque|
241
328
  if Rdkafka::Config.error_callback
242
- error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
329
+ instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
330
+ error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason, instance_name: instance_name)
243
331
  error.set_backtrace(caller)
244
332
  Rdkafka::Config.error_callback.call(error)
245
333
  end
@@ -266,9 +354,7 @@ module Rdkafka
266
354
  OAuthbearerTokenRefreshCallback = FFI::Function.new(
267
355
  :void, [:pointer, :string, :pointer]
268
356
  ) do |client_ptr, config, _opaque|
269
- if Rdkafka::Config.oauthbearer_token_refresh_callback
270
- Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
271
- end
357
+ Rdkafka::Config.oauthbearer_token_refresh_callback&.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
272
358
  end
273
359
 
274
360
  # Handle
@@ -296,7 +382,13 @@ module Rdkafka
296
382
  attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
297
383
  attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
298
384
  attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
385
+ # Non-blocking consumer poll variant (does not release GVL)
386
+ # More efficient for poll(0) calls in fiber schedulers.
387
+ attach_function :rd_kafka_consumer_poll_nb, :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: false
299
388
  attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
389
+ attach_function :rd_kafka_queue_get_consumer, [:pointer], :pointer
390
+ attach_function :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: true
391
+ attach_function :rd_kafka_consume_batch_queue_nb, :rd_kafka_consume_batch_queue, [:pointer, :int, :pointer, :size_t], :ssize_t, blocking: false
300
392
  attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
301
393
  attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
302
394
  attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
@@ -355,7 +447,6 @@ module Rdkafka
355
447
  attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
356
448
 
357
449
  # Producer
358
-
359
450
  RD_KAFKA_VTYPE_END = 0
360
451
  RD_KAFKA_VTYPE_TOPIC = 1
361
452
  RD_KAFKA_VTYPE_RKT = 2
@@ -377,16 +468,25 @@ module Rdkafka
377
468
  callback :delivery_cb, [:pointer, :pointer, :pointer], :void
378
469
  attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
379
470
 
380
- # Partitioner
381
- PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
382
- method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
471
+ # Hash mapping partitioner names to their FFI function symbols
472
+ # @return [Hash{String => Symbol}]
473
+ PARTITIONERS = %w[random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random].each_with_object({}) do |name, hsh|
474
+ method_name = :"rd_kafka_msg_partitioner_#{name}"
383
475
  attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
384
476
  hsh[name] = method_name
385
477
  end
386
478
 
479
+ # Calculates the partition for a message based on the partitioner
480
+ #
481
+ # @param topic_ptr [FFI::Pointer] pointer to the topic handle
482
+ # @param str [String] the partition key string
483
+ # @param partition_count [Integer, nil] number of partitions
484
+ # @param partitioner [String] name of the partitioner to use
485
+ # @return [Integer] partition number or RD_KAFKA_PARTITION_UA if unassigned
486
+ # @raise [Rdkafka::Config::ConfigError] when an unknown partitioner is specified
387
487
  def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
388
488
  # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
389
- return -1 unless partition_count&.nonzero?
489
+ return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
390
490
 
391
491
  str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
392
492
  method_name = PARTITIONERS.fetch(partitioner) do
@@ -397,8 +497,7 @@ module Rdkafka
397
497
  end
398
498
 
399
499
  # Create Topics
400
-
401
- RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
500
+ RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
402
501
  RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
403
502
 
404
503
  attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
@@ -409,8 +508,7 @@ module Rdkafka
409
508
  attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
410
509
 
411
510
  # Delete Topics
412
-
413
- RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
511
+ RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
414
512
  RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
415
513
 
416
514
  attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
@@ -430,7 +528,6 @@ module Rdkafka
430
528
  attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
431
529
 
432
530
  # Delete Group
433
-
434
531
  RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7 # rd_kafka_admin_op_t
435
532
  RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
436
533
 
@@ -442,7 +539,6 @@ module Rdkafka
442
539
 
443
540
  # Background Queue and Callback
444
541
 
445
- attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
446
542
  attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
447
543
  attach_function :rd_kafka_queue_destroy, [:pointer], :void
448
544
 
@@ -464,8 +560,7 @@ module Rdkafka
464
560
  attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
465
561
 
466
562
  # Create Acls
467
-
468
- RD_KAFKA_ADMIN_OP_CREATEACLS = 9
563
+ RD_KAFKA_ADMIN_OP_CREATEACLS = 9
469
564
  RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
470
565
 
471
566
  attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
@@ -473,8 +568,7 @@ module Rdkafka
473
568
  attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
474
569
 
475
570
  # Delete Acls
476
-
477
- RD_KAFKA_ADMIN_OP_DELETEACLS = 11
571
+ RD_KAFKA_ADMIN_OP_DELETEACLS = 11
478
572
  RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
479
573
 
480
574
  attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
@@ -484,8 +578,7 @@ module Rdkafka
484
578
  attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
485
579
 
486
580
  # Describe Acls
487
-
488
- RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
581
+ RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
489
582
  RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
490
583
 
491
584
  attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
@@ -501,45 +594,42 @@ module Rdkafka
501
594
  attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
502
595
  attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
503
596
  attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
504
- attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
505
- attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
597
+ attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
598
+ attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
506
599
  attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
507
600
 
508
601
  # rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
509
-
510
- RD_KAFKA_RESOURCE_ANY = 1
602
+ RD_KAFKA_RESOURCE_ANY = 1
511
603
  RD_KAFKA_RESOURCE_TOPIC = 2
512
604
  RD_KAFKA_RESOURCE_GROUP = 3
513
605
  RD_KAFKA_RESOURCE_BROKER = 4
514
606
  RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
515
607
 
516
608
  # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
517
-
518
- RD_KAFKA_RESOURCE_PATTERN_ANY = 1
519
- RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
520
- RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
609
+ RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
610
+ RD_KAFKA_RESOURCE_PATTERN_ANY = 1
611
+ RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
612
+ RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
521
613
  RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
522
614
 
523
615
  # rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
524
-
525
- RD_KAFKA_ACL_OPERATION_ANY = 1
526
- RD_KAFKA_ACL_OPERATION_ALL = 2
527
- RD_KAFKA_ACL_OPERATION_READ = 3
528
- RD_KAFKA_ACL_OPERATION_WRITE = 4
529
- RD_KAFKA_ACL_OPERATION_CREATE = 5
530
- RD_KAFKA_ACL_OPERATION_DELETE = 6
531
- RD_KAFKA_ACL_OPERATION_ALTER = 7
532
- RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
533
- RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
616
+ RD_KAFKA_ACL_OPERATION_ANY = 1
617
+ RD_KAFKA_ACL_OPERATION_ALL = 2
618
+ RD_KAFKA_ACL_OPERATION_READ = 3
619
+ RD_KAFKA_ACL_OPERATION_WRITE = 4
620
+ RD_KAFKA_ACL_OPERATION_CREATE = 5
621
+ RD_KAFKA_ACL_OPERATION_DELETE = 6
622
+ RD_KAFKA_ACL_OPERATION_ALTER = 7
623
+ RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
624
+ RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
534
625
  RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
535
- RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
626
+ RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
536
627
  RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
537
628
 
538
629
  # rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
539
-
540
- RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
541
- RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
542
- RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
630
+ RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
631
+ RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
632
+ RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
543
633
 
544
634
  # Extracting error details from Acl results
545
635
  attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
@@ -549,14 +639,13 @@ module Rdkafka
549
639
  attach_function :rd_kafka_event_error_string, [:pointer], :pointer
550
640
  attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
551
641
 
552
-
553
- # Extracting data from group results
554
- class NativeError < FFI::Struct # rd_kafka_error_t
642
+ # FFI struct for native error (rd_kafka_error_t)
643
+ class NativeError < FFI::Struct
555
644
  layout :code, :int32,
556
- :errstr, :pointer,
557
- :fatal, :u_int8_t,
558
- :retriable, :u_int8_t,
559
- :txn_requires_abort, :u_int8_t
645
+ :errstr, :pointer,
646
+ :fatal, :u_int8_t,
647
+ :retriable, :u_int8_t,
648
+ :txn_requires_abort, :u_int8_t
560
649
  end
561
650
 
562
651
  attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*