rdkafka 0.22.0-arm64-darwin

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/.github/CODEOWNERS +3 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci_linux_x86_64_gnu.yml +249 -0
  5. data/.github/workflows/ci_linux_x86_64_musl.yml +205 -0
  6. data/.github/workflows/ci_macos_arm64.yml +306 -0
  7. data/.github/workflows/push_linux_x86_64_gnu.yml +64 -0
  8. data/.github/workflows/push_linux_x86_64_musl.yml +77 -0
  9. data/.github/workflows/push_macos_arm64.yml +54 -0
  10. data/.github/workflows/push_ruby.yml +37 -0
  11. data/.github/workflows/verify-action-pins.yml +16 -0
  12. data/.gitignore +14 -0
  13. data/.rspec +2 -0
  14. data/.ruby-gemset +1 -0
  15. data/.ruby-version +1 -0
  16. data/.yardopts +2 -0
  17. data/CHANGELOG.md +249 -0
  18. data/Gemfile +5 -0
  19. data/MIT-LICENSE +22 -0
  20. data/README.md +178 -0
  21. data/Rakefile +96 -0
  22. data/docker-compose.yml +25 -0
  23. data/ext/README.md +19 -0
  24. data/ext/Rakefile +131 -0
  25. data/ext/build_common.sh +361 -0
  26. data/ext/build_linux_x86_64_gnu.sh +306 -0
  27. data/ext/build_linux_x86_64_musl.sh +763 -0
  28. data/ext/build_macos_arm64.sh +550 -0
  29. data/ext/librdkafka.dylib +0 -0
  30. data/lib/rdkafka/abstract_handle.rb +116 -0
  31. data/lib/rdkafka/admin/acl_binding_result.rb +51 -0
  32. data/lib/rdkafka/admin/config_binding_result.rb +30 -0
  33. data/lib/rdkafka/admin/config_resource_binding_result.rb +18 -0
  34. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  35. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  36. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  37. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  38. data/lib/rdkafka/admin/create_topic_handle.rb +29 -0
  39. data/lib/rdkafka/admin/create_topic_report.rb +24 -0
  40. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  41. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  42. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  43. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  44. data/lib/rdkafka/admin/delete_topic_handle.rb +29 -0
  45. data/lib/rdkafka/admin/delete_topic_report.rb +24 -0
  46. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  47. data/lib/rdkafka/admin/describe_acl_report.rb +24 -0
  48. data/lib/rdkafka/admin/describe_configs_handle.rb +33 -0
  49. data/lib/rdkafka/admin/describe_configs_report.rb +54 -0
  50. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +33 -0
  51. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +54 -0
  52. data/lib/rdkafka/admin.rb +833 -0
  53. data/lib/rdkafka/bindings.rb +565 -0
  54. data/lib/rdkafka/callbacks.rb +415 -0
  55. data/lib/rdkafka/config.rb +398 -0
  56. data/lib/rdkafka/consumer/headers.rb +79 -0
  57. data/lib/rdkafka/consumer/message.rb +86 -0
  58. data/lib/rdkafka/consumer/partition.rb +51 -0
  59. data/lib/rdkafka/consumer/topic_partition_list.rb +169 -0
  60. data/lib/rdkafka/consumer.rb +653 -0
  61. data/lib/rdkafka/error.rb +101 -0
  62. data/lib/rdkafka/helpers/oauth.rb +58 -0
  63. data/lib/rdkafka/helpers/time.rb +14 -0
  64. data/lib/rdkafka/metadata.rb +115 -0
  65. data/lib/rdkafka/native_kafka.rb +139 -0
  66. data/lib/rdkafka/producer/delivery_handle.rb +40 -0
  67. data/lib/rdkafka/producer/delivery_report.rb +46 -0
  68. data/lib/rdkafka/producer/partitions_count_cache.rb +216 -0
  69. data/lib/rdkafka/producer.rb +435 -0
  70. data/lib/rdkafka/version.rb +7 -0
  71. data/lib/rdkafka.rb +54 -0
  72. data/rdkafka.gemspec +65 -0
  73. data/renovate.json +92 -0
  74. data/spec/rdkafka/abstract_handle_spec.rb +117 -0
  75. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  76. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  77. data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
  78. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  79. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  80. data/spec/rdkafka/admin/delete_acl_report_spec.rb +72 -0
  81. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
  82. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  83. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  84. data/spec/rdkafka/admin/describe_acl_report_spec.rb +73 -0
  85. data/spec/rdkafka/admin_spec.rb +971 -0
  86. data/spec/rdkafka/bindings_spec.rb +199 -0
  87. data/spec/rdkafka/callbacks_spec.rb +20 -0
  88. data/spec/rdkafka/config_spec.rb +258 -0
  89. data/spec/rdkafka/consumer/headers_spec.rb +73 -0
  90. data/spec/rdkafka/consumer/message_spec.rb +139 -0
  91. data/spec/rdkafka/consumer/partition_spec.rb +57 -0
  92. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +248 -0
  93. data/spec/rdkafka/consumer_spec.rb +1274 -0
  94. data/spec/rdkafka/error_spec.rb +89 -0
  95. data/spec/rdkafka/metadata_spec.rb +79 -0
  96. data/spec/rdkafka/native_kafka_spec.rb +130 -0
  97. data/spec/rdkafka/producer/delivery_handle_spec.rb +45 -0
  98. data/spec/rdkafka/producer/delivery_report_spec.rb +25 -0
  99. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +359 -0
  100. data/spec/rdkafka/producer_spec.rb +1345 -0
  101. data/spec/spec_helper.rb +195 -0
  102. metadata +276 -0
@@ -0,0 +1,565 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ # @private
5
+ module Bindings
6
+ extend FFI::Library
7
+
8
+ def self.lib_extension
9
+ if RbConfig::CONFIG['host_os'] =~ /darwin/
10
+ 'dylib'
11
+ else
12
+ 'so'
13
+ end
14
+ end
15
+
16
+ ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
17
+
18
+ RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
19
+ RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
20
+ RD_KAFKA_RESP_ERR__STATE = -172
21
+ RD_KAFKA_RESP_ERR__NOENT = -156
22
+ RD_KAFKA_RESP_ERR_NO_ERROR = 0
23
+
24
+ RD_KAFKA_OFFSET_END = -1
25
+ RD_KAFKA_OFFSET_BEGINNING = -2
26
+ RD_KAFKA_OFFSET_STORED = -1000
27
+ RD_KAFKA_OFFSET_INVALID = -1001
28
+
29
+ EMPTY_HASH = {}.freeze
30
+
31
+ class SizePtr < FFI::Struct
32
+ layout :value, :size_t
33
+ end
34
+
35
+ # This function comes from our patch on top of librdkafka. It allows os to load all the
36
+ # librdkafka components without initializing the client
37
+ # @see https://github.com/confluentinc/librdkafka/issues/4590
38
+ attach_function :rd_kafka_global_init, [], :void
39
+
40
+ # Polling
41
+
42
+ attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
43
+ attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
44
+ attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
45
+
46
+ # Metadata
47
+
48
+ attach_function :rd_kafka_name, [:pointer], :string
49
+ attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
50
+ attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
51
+ attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
52
+ attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
53
+
54
+ # Message struct
55
+
56
+ class Message < FFI::Struct
57
+ layout :err, :int,
58
+ :rkt, :pointer,
59
+ :partition, :int32,
60
+ :payload, :pointer,
61
+ :len, :size_t,
62
+ :key, :pointer,
63
+ :key_len, :size_t,
64
+ :offset, :int64,
65
+ :_private, :pointer
66
+ end
67
+
68
+ attach_function :rd_kafka_message_destroy, [:pointer], :void
69
+ attach_function :rd_kafka_message_timestamp, [:pointer, :pointer], :int64
70
+ attach_function :rd_kafka_topic_new, [:pointer, :string, :pointer], :pointer
71
+ attach_function :rd_kafka_topic_destroy, [:pointer], :pointer
72
+ attach_function :rd_kafka_topic_name, [:pointer], :string
73
+
74
+ # TopicPartition ad TopicPartitionList structs
75
+
76
+ class TopicPartition < FFI::Struct
77
+ layout :topic, :string,
78
+ :partition, :int32,
79
+ :offset, :int64,
80
+ :metadata, :pointer,
81
+ :metadata_size, :size_t,
82
+ :opaque, :pointer,
83
+ :err, :int,
84
+ :_private, :pointer
85
+ end
86
+
87
+ class TopicPartitionList < FFI::Struct
88
+ layout :cnt, :int,
89
+ :size, :int,
90
+ :elems, :pointer
91
+ end
92
+
93
+ attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
94
+ attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
95
+ attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
96
+ attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
97
+ attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
98
+
99
+ # Configs management
100
+ #
101
+ # Structs for management of configurations
102
+ # Each configuration is attached to a resource and one resource can have many configuration
103
+ # details. Each resource will also have separate errors results if obtaining configuration
104
+ # was not possible for any reason
105
+ class ConfigResource < FFI::Struct
106
+ layout :type, :int,
107
+ :name, :string
108
+ end
109
+
110
+ attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
111
+ attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
112
+ attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
113
+ attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
114
+ attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
115
+ attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
116
+ attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
117
+ attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
118
+ attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
119
+ attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
120
+ attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
121
+ attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
122
+ attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
123
+ attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
124
+ attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
125
+ attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
126
+ attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
127
+ attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
128
+ attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
129
+
130
+ RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
131
+ RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
132
+
133
+ RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
134
+ RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
135
+
136
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
137
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
138
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
139
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
140
+
141
+ # Errors
142
+ class NativeErrorDesc < FFI::Struct
143
+ layout :code, :int,
144
+ :name, :pointer,
145
+ :desc, :pointer
146
+ end
147
+
148
+ attach_function :rd_kafka_err2name, [:int], :string
149
+ attach_function :rd_kafka_err2str, [:int], :string
150
+ attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
151
+
152
+ # Configuration
153
+
154
+ enum :kafka_config_response, [
155
+ :config_unknown, -2,
156
+ :config_invalid, -1,
157
+ :config_ok, 0
158
+ ]
159
+
160
+ attach_function :rd_kafka_conf_new, [], :pointer
161
+ attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
162
+ callback :log_cb, [:pointer, :int, :string, :string], :void
163
+ attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
164
+ attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
165
+ callback :stats_cb, [:pointer, :string, :int, :pointer], :int
166
+ attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
167
+ callback :error_cb, [:pointer, :int, :string, :pointer], :void
168
+ attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
169
+ attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
170
+ callback :oauthbearer_token_refresh_cb, [:pointer, :string, :pointer], :void
171
+ attach_function :rd_kafka_conf_set_oauthbearer_token_refresh_cb, [:pointer, :oauthbearer_token_refresh_cb], :void
172
+ attach_function :rd_kafka_oauthbearer_set_token, [:pointer, :string, :int64, :pointer, :pointer, :int, :pointer, :int], :int
173
+ attach_function :rd_kafka_oauthbearer_set_token_failure, [:pointer, :string], :int
174
+ # Log queue
175
+ attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
176
+ attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
177
+ # Per topic configs
178
+ attach_function :rd_kafka_topic_conf_new, [], :pointer
179
+ attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
180
+
181
+ LogCallback = FFI::Function.new(
182
+ :void, [:pointer, :int, :string, :string]
183
+ ) do |_client_ptr, level, _level_string, line|
184
+ severity = case level
185
+ when 0, 1, 2
186
+ Logger::FATAL
187
+ when 3
188
+ Logger::ERROR
189
+ when 4
190
+ Logger::WARN
191
+ when 5, 6
192
+ Logger::INFO
193
+ when 7
194
+ Logger::DEBUG
195
+ else
196
+ Logger::UNKNOWN
197
+ end
198
+
199
+ Rdkafka::Config.ensure_log_thread
200
+ Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
201
+ end
202
+
203
+ StatsCallback = FFI::Function.new(
204
+ :int, [:pointer, :string, :int, :pointer]
205
+ ) do |_client_ptr, json, _json_len, _opaque|
206
+ if Rdkafka::Config.statistics_callback
207
+ stats = JSON.parse(json)
208
+
209
+ # If user requested statistics callbacks, we can use the statistics data to get the
210
+ # partitions count for each topic when this data is published. That way we do not have
211
+ # to query this information when user is using `partition_key`. This takes around 0.02ms
212
+ # every statistics interval period (most likely every 5 seconds) and saves us from making
213
+ # any queries to the cluster for the partition count.
214
+ #
215
+ # One edge case is if user would set the `statistics.interval.ms` much higher than the
216
+ # default current partition count refresh (30 seconds). This is taken care of as the lack
217
+ # of reporting to the partitions cache will cause cache expire and blocking refresh.
218
+ #
219
+ # If user sets `topic.metadata.refresh.interval.ms` too high this is on the user.
220
+ #
221
+ # Since this cache is shared, having few consumers and/or producers in one process will
222
+ # automatically improve the querying times even with low refresh times.
223
+ (stats['topics'] || EMPTY_HASH).each do |topic_name, details|
224
+ partitions_count = details['partitions'].keys.reject { |k| k == '-1' }.size
225
+
226
+ next unless partitions_count.positive?
227
+
228
+ Rdkafka::Producer.partitions_count_cache.set(topic_name, partitions_count)
229
+ end
230
+
231
+ Rdkafka::Config.statistics_callback.call(stats)
232
+ end
233
+
234
+ # Return 0 so librdkafka frees the json string
235
+ 0
236
+ end
237
+
238
+ ErrorCallback = FFI::Function.new(
239
+ :void, [:pointer, :int, :string, :pointer]
240
+ ) do |_client_prr, err_code, reason, _opaque|
241
+ if Rdkafka::Config.error_callback
242
+ error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
243
+ error.set_backtrace(caller)
244
+ Rdkafka::Config.error_callback.call(error)
245
+ end
246
+ end
247
+
248
+ # The OAuth callback is currently global and contextless.
249
+ # This means that the callback will be called for all instances, and the callback must be able to determine to which instance it is associated.
250
+ # The instance name will be provided in the callback, allowing the callback to reference the correct instance.
251
+ #
252
+ # An example of how to use the instance name in the callback is given below.
253
+ # The `refresh_token` is configured as the `oauthbearer_token_refresh_callback`.
254
+ # `instances` is a map of client names to client instances, maintained by the user.
255
+ #
256
+ # ```
257
+ # def refresh_token(config, client_name)
258
+ # client = instances[client_name]
259
+ # client.oauthbearer_set_token(
260
+ # token: 'new-token-value',
261
+ # lifetime_ms: token-lifetime-ms,
262
+ # principal_name: 'principal-name'
263
+ # )
264
+ # end
265
+ # ```
266
+ OAuthbearerTokenRefreshCallback = FFI::Function.new(
267
+ :void, [:pointer, :string, :pointer]
268
+ ) do |client_ptr, config, _opaque|
269
+ if Rdkafka::Config.oauthbearer_token_refresh_callback
270
+ Rdkafka::Config.oauthbearer_token_refresh_callback.call(config, Rdkafka::Bindings.rd_kafka_name(client_ptr))
271
+ end
272
+ end
273
+
274
+ # Handle
275
+
276
+ enum :kafka_type, [
277
+ :rd_kafka_producer,
278
+ :rd_kafka_consumer
279
+ ]
280
+
281
+ attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
282
+
283
+ attach_function :rd_kafka_destroy, [:pointer], :void
284
+
285
+ # Consumer
286
+
287
+ attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
288
+ attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
289
+ attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
290
+ attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
291
+ attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
292
+ attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
293
+ attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
294
+ attach_function :rd_kafka_assignment_lost, [:pointer], :int, blocking: true
295
+ attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
296
+ attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
297
+ attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
298
+ attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
299
+ attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
300
+ attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
301
+ attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
302
+ attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
303
+ attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
304
+ attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
305
+ attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
306
+ # those two are used for eos support
307
+ attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
308
+ attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
309
+
310
+ # Headers
311
+ attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
312
+ attach_function :rd_kafka_message_headers, [:pointer, :pointer], :int
313
+
314
+ # Rebalance
315
+
316
+ callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
317
+ attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
318
+
319
+ RebalanceCallback = FFI::Function.new(
320
+ :void, [:pointer, :int, :pointer, :pointer]
321
+ ) do |client_ptr, code, partitions_ptr, opaque_ptr|
322
+ case code
323
+ when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
324
+ if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
325
+ Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
326
+ else
327
+ Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
328
+ end
329
+ else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
330
+ if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
331
+ Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
332
+ else
333
+ Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
334
+ end
335
+ end
336
+
337
+ opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
338
+ return unless opaque
339
+
340
+ tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
341
+ begin
342
+ case code
343
+ when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
344
+ opaque.call_on_partitions_assigned(tpl)
345
+ when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
346
+ opaque.call_on_partitions_revoked(tpl)
347
+ end
348
+ rescue Exception => err
349
+ Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
350
+ end
351
+ end
352
+
353
+ # Stats
354
+
355
+ attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
356
+
357
+ # Producer
358
+
359
+ RD_KAFKA_VTYPE_END = 0
360
+ RD_KAFKA_VTYPE_TOPIC = 1
361
+ RD_KAFKA_VTYPE_RKT = 2
362
+ RD_KAFKA_VTYPE_PARTITION = 3
363
+ RD_KAFKA_VTYPE_VALUE = 4
364
+ RD_KAFKA_VTYPE_KEY = 5
365
+ RD_KAFKA_VTYPE_OPAQUE = 6
366
+ RD_KAFKA_VTYPE_MSGFLAGS = 7
367
+ RD_KAFKA_VTYPE_TIMESTAMP = 8
368
+ RD_KAFKA_VTYPE_HEADER = 9
369
+ RD_KAFKA_VTYPE_HEADERS = 10
370
+ RD_KAFKA_PURGE_F_QUEUE = 1
371
+ RD_KAFKA_PURGE_F_INFLIGHT = 2
372
+
373
+ RD_KAFKA_MSG_F_COPY = 0x2
374
+
375
+ attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
376
+ attach_function :rd_kafka_purge, [:pointer, :int], :int, blocking: true
377
+ callback :delivery_cb, [:pointer, :pointer, :pointer], :void
378
+ attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
379
+
380
+ # Partitioner
381
+ PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
382
+ method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
383
+ attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
384
+ hsh[name] = method_name
385
+ end
386
+
387
+ def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
388
+ # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
389
+ return -1 unless partition_count&.nonzero?
390
+
391
+ str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
392
+ method_name = PARTITIONERS.fetch(partitioner) do
393
+ raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
394
+ end
395
+
396
+ public_send(method_name, topic_ptr, str_ptr, str.size, partition_count, nil, nil)
397
+ end
398
+
399
+ # Create Topics
400
+
401
+ RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
402
+ RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
403
+
404
+ attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
405
+ attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
406
+ attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
407
+ attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
408
+ attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
409
+ attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
410
+
411
+ # Delete Topics
412
+
413
+ RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
414
+ RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
415
+
416
+ attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
417
+ attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
418
+ attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
419
+ attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
420
+ attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
421
+
422
+ # Create partitions
423
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3
424
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT = 102
425
+
426
+ attach_function :rd_kafka_CreatePartitions, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
427
+ attach_function :rd_kafka_NewPartitions_new, %i[pointer size_t pointer size_t], :pointer
428
+ attach_function :rd_kafka_NewPartitions_destroy, [:pointer], :void
429
+ attach_function :rd_kafka_event_CreatePartitions_result, [:pointer], :pointer
430
+ attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
431
+
432
+ # Delete Group
433
+
434
+ RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7 # rd_kafka_admin_op_t
435
+ RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
436
+
437
+ attach_function :rd_kafka_DeleteGroups, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
438
+ attach_function :rd_kafka_DeleteGroup_new, [:pointer], :pointer, blocking: true
439
+ attach_function :rd_kafka_DeleteGroup_destroy, [:pointer], :void, blocking: true
440
+ attach_function :rd_kafka_event_DeleteGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteGroups_result_t*
441
+ attach_function :rd_kafka_DeleteGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteGroups_result_t*, size_t* => rd_kafka_group_result_t**
442
+
443
+ # Background Queue and Callback
444
+
445
+ attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
446
+ attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
447
+ attach_function :rd_kafka_queue_destroy, [:pointer], :void
448
+
449
+ # Admin Options
450
+
451
+ attach_function :rd_kafka_AdminOptions_new, [:pointer, :int32], :pointer
452
+ attach_function :rd_kafka_AdminOptions_set_opaque, [:pointer, :pointer], :void
453
+ attach_function :rd_kafka_AdminOptions_destroy, [:pointer], :void
454
+
455
+ # Extracting data from event types
456
+
457
+ attach_function :rd_kafka_event_type, [:pointer], :int32
458
+ attach_function :rd_kafka_event_opaque, [:pointer], :pointer
459
+
460
+ # Extracting data from topic results
461
+
462
+ attach_function :rd_kafka_topic_result_error, [:pointer], :int32
463
+ attach_function :rd_kafka_topic_result_error_string, [:pointer], :pointer
464
+ attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
465
+
466
+ # Create Acls
467
+
468
+ RD_KAFKA_ADMIN_OP_CREATEACLS = 9
469
+ RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
470
+
471
+ attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
472
+ attach_function :rd_kafka_event_CreateAcls_result, [:pointer], :pointer
473
+ attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
474
+
475
+ # Delete Acls
476
+
477
+ RD_KAFKA_ADMIN_OP_DELETEACLS = 11
478
+ RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
479
+
480
+ attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
481
+ attach_function :rd_kafka_event_DeleteAcls_result, [:pointer], :pointer
482
+ attach_function :rd_kafka_DeleteAcls_result_responses, [:pointer, :pointer], :pointer
483
+ attach_function :rd_kafka_DeleteAcls_result_response_error, [:pointer], :pointer
484
+ attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
485
+
486
+ # Describe Acls
487
+
488
+ RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
489
+ RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
490
+
491
+ attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
492
+ attach_function :rd_kafka_event_DescribeAcls_result, [:pointer], :pointer
493
+ attach_function :rd_kafka_DescribeAcls_result_acls, [:pointer, :pointer], :pointer
494
+
495
+ # Acl Bindings
496
+
497
+ attach_function :rd_kafka_AclBinding_restype, [:pointer], :int32
498
+ attach_function :rd_kafka_AclBinding_name, [:pointer], :pointer
499
+ attach_function :rd_kafka_AclBinding_resource_pattern_type, [:pointer], :int32
500
+ attach_function :rd_kafka_AclBinding_principal, [:pointer], :pointer
501
+ attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
502
+ attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
503
+ attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
504
+ attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
505
+ attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
506
+ attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
507
+
508
+ # rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
509
+
510
+ RD_KAFKA_RESOURCE_ANY = 1
511
+ RD_KAFKA_RESOURCE_TOPIC = 2
512
+ RD_KAFKA_RESOURCE_GROUP = 3
513
+ RD_KAFKA_RESOURCE_BROKER = 4
514
+ RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
515
+
516
+ # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
517
+
518
+ RD_KAFKA_RESOURCE_PATTERN_ANY = 1
519
+ RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
520
+ RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
521
+ RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
522
+
523
+ # rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
524
+
525
+ RD_KAFKA_ACL_OPERATION_ANY = 1
526
+ RD_KAFKA_ACL_OPERATION_ALL = 2
527
+ RD_KAFKA_ACL_OPERATION_READ = 3
528
+ RD_KAFKA_ACL_OPERATION_WRITE = 4
529
+ RD_KAFKA_ACL_OPERATION_CREATE = 5
530
+ RD_KAFKA_ACL_OPERATION_DELETE = 6
531
+ RD_KAFKA_ACL_OPERATION_ALTER = 7
532
+ RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
533
+ RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
534
+ RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
535
+ RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
536
+ RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
537
+
538
+ # rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
539
+
540
+ RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
541
+ RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
542
+ RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
543
+
544
+ # Extracting error details from Acl results
545
+ attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
546
+ attach_function :rd_kafka_error_code, [:pointer], :int32
547
+ attach_function :rd_kafka_error_string, [:pointer], :pointer
548
+ attach_function :rd_kafka_event_error, [:pointer], :int32
549
+ attach_function :rd_kafka_event_error_string, [:pointer], :pointer
550
+ attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
551
+
552
+
553
+ # Extracting data from group results
554
+ class NativeError < FFI::Struct # rd_kafka_error_t
555
+ layout :code, :int32,
556
+ :errstr, :pointer,
557
+ :fatal, :u_int8_t,
558
+ :retriable, :u_int8_t,
559
+ :txn_requires_abort, :u_int8_t
560
+ end
561
+
562
+ attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
563
+ attach_function :rd_kafka_group_result_name, [:pointer], :pointer
564
+ end
565
+ end