rdkafka 0.12.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +58 -0
  5. data/.gitignore +4 -0
  6. data/.rspec +1 -0
  7. data/.ruby-gemset +1 -0
  8. data/.ruby-version +1 -0
  9. data/CHANGELOG.md +141 -93
  10. data/Gemfile +2 -0
  11. data/{LICENSE → MIT-LICENSE} +2 -1
  12. data/README.md +64 -29
  13. data/Rakefile +2 -0
  14. data/certs/cert_chain.pem +26 -0
  15. data/docker-compose.yml +18 -15
  16. data/ext/README.md +1 -1
  17. data/ext/Rakefile +3 -1
  18. data/lib/rdkafka/abstract_handle.rb +41 -25
  19. data/lib/rdkafka/admin/acl_binding_result.rb +37 -0
  20. data/lib/rdkafka/admin/create_acl_handle.rb +28 -0
  21. data/lib/rdkafka/admin/create_acl_report.rb +24 -0
  22. data/lib/rdkafka/admin/create_partitions_handle.rb +27 -0
  23. data/lib/rdkafka/admin/create_partitions_report.rb +6 -0
  24. data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
  25. data/lib/rdkafka/admin/create_topic_report.rb +2 -0
  26. data/lib/rdkafka/admin/delete_acl_handle.rb +30 -0
  27. data/lib/rdkafka/admin/delete_acl_report.rb +23 -0
  28. data/lib/rdkafka/admin/delete_groups_handle.rb +28 -0
  29. data/lib/rdkafka/admin/delete_groups_report.rb +24 -0
  30. data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
  31. data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
  32. data/lib/rdkafka/admin/describe_acl_handle.rb +30 -0
  33. data/lib/rdkafka/admin/describe_acl_report.rb +23 -0
  34. data/lib/rdkafka/admin.rb +494 -35
  35. data/lib/rdkafka/bindings.rb +175 -40
  36. data/lib/rdkafka/callbacks.rb +194 -1
  37. data/lib/rdkafka/config.rb +62 -25
  38. data/lib/rdkafka/consumer/headers.rb +24 -9
  39. data/lib/rdkafka/consumer/message.rb +3 -1
  40. data/lib/rdkafka/consumer/partition.rb +2 -0
  41. data/lib/rdkafka/consumer/topic_partition_list.rb +13 -8
  42. data/lib/rdkafka/consumer.rb +219 -102
  43. data/lib/rdkafka/error.rb +15 -0
  44. data/lib/rdkafka/helpers/time.rb +14 -0
  45. data/lib/rdkafka/metadata.rb +25 -2
  46. data/lib/rdkafka/native_kafka.rb +120 -0
  47. data/lib/rdkafka/producer/delivery_handle.rb +5 -2
  48. data/lib/rdkafka/producer/delivery_report.rb +9 -2
  49. data/lib/rdkafka/producer.rb +117 -17
  50. data/lib/rdkafka/version.rb +5 -3
  51. data/lib/rdkafka.rb +24 -2
  52. data/rdkafka.gemspec +19 -3
  53. data/renovate.json +6 -0
  54. data/spec/rdkafka/abstract_handle_spec.rb +1 -1
  55. data/spec/rdkafka/admin/create_acl_handle_spec.rb +56 -0
  56. data/spec/rdkafka/admin/create_acl_report_spec.rb +18 -0
  57. data/spec/rdkafka/admin/create_topic_handle_spec.rb +1 -1
  58. data/spec/rdkafka/admin/create_topic_report_spec.rb +1 -1
  59. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +85 -0
  60. data/spec/rdkafka/admin/delete_acl_report_spec.rb +71 -0
  61. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +1 -1
  62. data/spec/rdkafka/admin/delete_topic_report_spec.rb +1 -1
  63. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +85 -0
  64. data/spec/rdkafka/admin/describe_acl_report_spec.rb +72 -0
  65. data/spec/rdkafka/admin_spec.rb +209 -5
  66. data/spec/rdkafka/bindings_spec.rb +2 -1
  67. data/spec/rdkafka/callbacks_spec.rb +1 -1
  68. data/spec/rdkafka/config_spec.rb +24 -3
  69. data/spec/rdkafka/consumer/headers_spec.rb +60 -0
  70. data/spec/rdkafka/consumer/message_spec.rb +1 -1
  71. data/spec/rdkafka/consumer/partition_spec.rb +1 -1
  72. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +20 -1
  73. data/spec/rdkafka/consumer_spec.rb +332 -61
  74. data/spec/rdkafka/error_spec.rb +1 -1
  75. data/spec/rdkafka/metadata_spec.rb +4 -3
  76. data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -35
  77. data/spec/rdkafka/producer/delivery_handle_spec.rb +4 -1
  78. data/spec/rdkafka/producer/delivery_report_spec.rb +7 -3
  79. data/spec/rdkafka/producer_spec.rb +208 -20
  80. data/spec/spec_helper.rb +20 -2
  81. data.tar.gz.sig +3 -0
  82. metadata +79 -16
  83. metadata.gz.sig +3 -0
  84. data/.semaphore/semaphore.yml +0 -23
  85. data/bin/console +0 -11
  86. data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,6 +1,4 @@
1
- require "ffi"
2
- require "json"
3
- require "logger"
1
+ # frozen_string_literal: true
4
2
 
5
3
  module Rdkafka
6
4
  # @private
@@ -15,7 +13,7 @@ module Rdkafka
15
13
  end
16
14
  end
17
15
 
18
- ffi_lib File.join(File.dirname(__FILE__), "../../ext/librdkafka.#{lib_extension}")
16
+ ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
19
17
 
20
18
  RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
21
19
  RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
@@ -33,15 +31,17 @@ module Rdkafka
33
31
 
34
32
  # Polling
35
33
 
34
+ attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
36
35
  attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
37
36
  attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
38
37
 
39
38
  # Metadata
40
39
 
41
- attach_function :rd_kafka_memberid, [:pointer], :string
42
- attach_function :rd_kafka_clusterid, [:pointer], :string
43
- attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
44
- attach_function :rd_kafka_metadata_destroy, [:pointer], :void
40
+ attach_function :rd_kafka_name, [:pointer], :string, blocking: true
41
+ attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
42
+ attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
43
+ attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
44
+ attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
45
45
 
46
46
  # Message struct
47
47
 
@@ -110,6 +110,7 @@ module Rdkafka
110
110
  attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
111
111
  callback :error_cb, [:pointer, :int, :string, :pointer], :void
112
112
  attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
113
+ attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
113
114
 
114
115
  # Log queue
115
116
  attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -153,6 +154,7 @@ module Rdkafka
153
154
  ) do |_client_prr, err_code, reason, _opaque|
154
155
  if Rdkafka::Config.error_callback
155
156
  error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
157
+ error.set_backtrace(caller)
156
158
  Rdkafka::Config.error_callback.call(error)
157
159
  end
158
160
  end
@@ -165,24 +167,30 @@ module Rdkafka
165
167
  ]
166
168
 
167
169
  attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
170
+
168
171
  attach_function :rd_kafka_destroy, [:pointer], :void
169
172
 
170
173
  # Consumer
171
174
 
172
- attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
173
- attach_function :rd_kafka_unsubscribe, [:pointer], :int
174
- attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
175
- attach_function :rd_kafka_assign, [:pointer, :pointer], :int
176
- attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
177
- attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
175
+ attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
176
+ attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
177
+ attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
178
+ attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
179
+ attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
180
+ attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
181
+ attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
182
+ attach_function :rd_kafka_assignment_lost, [:pointer], :int, blocking: true
183
+ attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
178
184
  attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
179
- attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
185
+ attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
180
186
  attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
181
187
  attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
182
- attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
183
- attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
184
- attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
185
- attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
188
+ attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
189
+ attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
190
+ attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
191
+ attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
192
+ attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
193
+ attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
186
194
 
187
195
  # Headers
188
196
  attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -191,30 +199,36 @@ module Rdkafka
191
199
  # Rebalance
192
200
 
193
201
  callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
194
- attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
202
+ attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
195
203
 
196
204
  RebalanceCallback = FFI::Function.new(
197
205
  :void, [:pointer, :int, :pointer, :pointer]
198
206
  ) do |client_ptr, code, partitions_ptr, opaque_ptr|
199
207
  case code
200
208
  when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
201
- Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
209
+ if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
210
+ Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
211
+ else
212
+ Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
213
+ end
202
214
  else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
203
- Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
215
+ if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
216
+ Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
217
+ else
218
+ Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
219
+ end
204
220
  end
205
221
 
206
222
  opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
207
223
  return unless opaque
208
224
 
209
225
  tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
210
- consumer = Rdkafka::Consumer.new(client_ptr)
211
-
212
226
  begin
213
227
  case code
214
228
  when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
215
- opaque.call_on_partitions_assigned(consumer, tpl)
229
+ opaque.call_on_partitions_assigned(tpl)
216
230
  when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
217
- opaque.call_on_partitions_revoked(consumer, tpl)
231
+ opaque.call_on_partitions_revoked(tpl)
218
232
  end
219
233
  rescue Exception => err
220
234
  Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
@@ -238,10 +252,13 @@ module Rdkafka
238
252
  RD_KAFKA_VTYPE_TIMESTAMP = 8
239
253
  RD_KAFKA_VTYPE_HEADER = 9
240
254
  RD_KAFKA_VTYPE_HEADERS = 10
255
+ RD_KAFKA_PURGE_F_QUEUE = 1
256
+ RD_KAFKA_PURGE_F_INFLIGHT = 2
241
257
 
242
258
  RD_KAFKA_MSG_F_COPY = 0x2
243
259
 
244
- attach_function :rd_kafka_producev, [:pointer, :varargs], :int
260
+ attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
261
+ attach_function :rd_kafka_purge, [:pointer, :int], :int, blocking: true
245
262
  callback :delivery_cb, [:pointer, :pointer, :pointer], :void
246
263
  attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
247
264
 
@@ -256,11 +273,11 @@ module Rdkafka
256
273
  # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
257
274
  return -1 unless partition_count&.nonzero?
258
275
 
259
- str_ptr = FFI::MemoryPointer.from_string(str)
276
+ str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
260
277
  method_name = PARTITIONERS.fetch(partitioner_name) do
261
278
  raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
262
279
  end
263
- public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
280
+ public_send(method_name, nil, str_ptr, str.size > 0 ? str.size : 1, partition_count, nil, nil)
264
281
  end
265
282
 
266
283
  # Create Topics
@@ -268,23 +285,44 @@ module Rdkafka
268
285
  RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
269
286
  RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
270
287
 
271
- attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
272
- attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
273
- attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
274
- attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
275
- attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
276
- attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
288
+ attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
289
+ attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
290
+ attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
291
+ attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
292
+ attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
293
+ attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
277
294
 
278
295
  # Delete Topics
279
296
 
280
297
  RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
281
298
  RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
282
299
 
283
- attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
284
- attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
285
- attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
286
- attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
287
- attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
300
+ attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
301
+ attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
302
+ attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
303
+ attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
304
+ attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
305
+
306
+ # Create partitions
307
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3
308
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT = 102
309
+
310
+ attach_function :rd_kafka_CreatePartitions, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
311
+ attach_function :rd_kafka_NewPartitions_new, %i[pointer size_t pointer size_t], :pointer
312
+ attach_function :rd_kafka_NewPartitions_destroy, [:pointer], :void
313
+ attach_function :rd_kafka_event_CreatePartitions_result, [:pointer], :pointer
314
+ attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
315
+
316
+ # Delete Group
317
+
318
+ RD_KAFKA_ADMIN_OP_DELETEGROUPS = 7 # rd_kafka_admin_op_t
319
+ RD_KAFKA_EVENT_DELETEGROUPS_RESULT = 106 # rd_kafka_event_type_t
320
+
321
+ attach_function :rd_kafka_DeleteGroups, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
322
+ attach_function :rd_kafka_DeleteGroup_new, [:pointer], :pointer, blocking: true
323
+ attach_function :rd_kafka_DeleteGroup_destroy, [:pointer], :void, blocking: true
324
+ attach_function :rd_kafka_event_DeleteGroups_result, [:pointer], :pointer, blocking: true # rd_kafka_event_t* => rd_kafka_DeleteGroups_result_t*
325
+ attach_function :rd_kafka_DeleteGroups_result_groups, [:pointer, :pointer], :pointer, blocking: true # rd_kafka_DeleteGroups_result_t*, size_t* => rd_kafka_group_result_t**
288
326
 
289
327
  # Background Queue and Callback
290
328
 
@@ -308,5 +346,102 @@ module Rdkafka
308
346
  attach_function :rd_kafka_topic_result_error, [:pointer], :int32
309
347
  attach_function :rd_kafka_topic_result_error_string, [:pointer], :pointer
310
348
  attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
349
+
350
+ # Create Acls
351
+
352
+ RD_KAFKA_ADMIN_OP_CREATEACLS = 9
353
+ RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
354
+
355
+ attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
356
+ attach_function :rd_kafka_event_CreateAcls_result, [:pointer], :pointer
357
+ attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
358
+
359
+ # Delete Acls
360
+
361
+ RD_KAFKA_ADMIN_OP_DELETEACLS = 11
362
+ RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
363
+
364
+ attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
365
+ attach_function :rd_kafka_event_DeleteAcls_result, [:pointer], :pointer
366
+ attach_function :rd_kafka_DeleteAcls_result_responses, [:pointer, :pointer], :pointer
367
+ attach_function :rd_kafka_DeleteAcls_result_response_error, [:pointer], :pointer
368
+ attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
369
+
370
+ # Describe Acls
371
+
372
+ RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
373
+ RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
374
+
375
+ attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
376
+ attach_function :rd_kafka_event_DescribeAcls_result, [:pointer], :pointer
377
+ attach_function :rd_kafka_DescribeAcls_result_acls, [:pointer, :pointer], :pointer
378
+
379
+ # Acl Bindings
380
+
381
+ attach_function :rd_kafka_AclBinding_restype, [:pointer], :int32
382
+ attach_function :rd_kafka_AclBinding_name, [:pointer], :pointer
383
+ attach_function :rd_kafka_AclBinding_resource_pattern_type, [:pointer], :int32
384
+ attach_function :rd_kafka_AclBinding_principal, [:pointer], :pointer
385
+ attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
386
+ attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
387
+ attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
388
+ attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
389
+ attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t ], :pointer
390
+ attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
391
+
392
+ # rd_kafka_ResourceType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7307
393
+
394
+ RD_KAFKA_RESOURCE_ANY = 1
395
+ RD_KAFKA_RESOURCE_TOPIC = 2
396
+ RD_KAFKA_RESOURCE_GROUP = 3
397
+
398
+ # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
399
+
400
+ RD_KAFKA_RESOURCE_PATTERN_ANY = 1
401
+ RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
402
+ RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
403
+ RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
404
+
405
+ # rd_kafka_AclOperation_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8403
406
+
407
+ RD_KAFKA_ACL_OPERATION_ANY = 1
408
+ RD_KAFKA_ACL_OPERATION_ALL = 2
409
+ RD_KAFKA_ACL_OPERATION_READ = 3
410
+ RD_KAFKA_ACL_OPERATION_WRITE = 4
411
+ RD_KAFKA_ACL_OPERATION_CREATE = 5
412
+ RD_KAFKA_ACL_OPERATION_DELETE = 6
413
+ RD_KAFKA_ACL_OPERATION_ALTER = 7
414
+ RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
415
+ RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
416
+ RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
417
+ RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
418
+ RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
419
+
420
+ # rd_kafka_AclPermissionType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L8435
421
+
422
+ RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
423
+ RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
424
+ RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
425
+
426
+ # Extracting error details from Acl results
427
+ attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
428
+ attach_function :rd_kafka_error_code, [:pointer], :int32
429
+ attach_function :rd_kafka_error_string, [:pointer], :pointer
430
+ attach_function :rd_kafka_event_error, [:pointer], :int32
431
+ attach_function :rd_kafka_event_error_string, [:pointer], :pointer
432
+ attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
433
+
434
+
435
+ # Extracting data from group results
436
+ class NativeError < FFI::Struct # rd_kafka_error_t
437
+ layout :code, :int32,
438
+ :errstr, :pointer,
439
+ :fatal, :uint8_t,
440
+ :retriable, :uint8_t,
441
+ :txn_requires_abort, :uint8_t
442
+ end
443
+
444
+ attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*
445
+ attach_function :rd_kafka_group_result_name, [:pointer], :pointer
311
446
  end
312
447
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rdkafka
2
4
  module Callbacks
3
5
 
@@ -21,6 +23,96 @@ module Rdkafka
21
23
  end
22
24
  end
23
25
 
26
+ class GroupResult
27
+ attr_reader :result_error, :error_string, :result_name
28
+ def initialize(group_result_pointer)
29
+ native_error = Rdkafka::Bindings.rd_kafka_group_result_error(group_result_pointer)
30
+
31
+ if native_error.null?
32
+ @result_error = 0
33
+ @error_string = FFI::Pointer::NULL
34
+ else
35
+ @result_error = native_error[:code]
36
+ @error_string = native_error[:errstr]
37
+ end
38
+
39
+ @result_name = Rdkafka::Bindings.rd_kafka_group_result_name(group_result_pointer)
40
+ end
41
+ def self.create_group_results_from_array(count, array_pointer)
42
+ (1..count).map do |index|
43
+ result_pointer = (array_pointer + (index - 1)).read_pointer
44
+ new(result_pointer)
45
+ end
46
+ end
47
+ end
48
+
49
+ # Extracts attributes of rd_kafka_acl_result_t
50
+ #
51
+ # @private
52
+ class CreateAclResult
53
+ attr_reader :result_error, :error_string
54
+
55
+ def initialize(acl_result_pointer)
56
+ rd_kafka_error_pointer = Bindings.rd_kafka_acl_result_error(acl_result_pointer)
57
+ @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
58
+ @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
59
+ end
60
+
61
+ def self.create_acl_results_from_array(count, array_pointer)
62
+ (1..count).map do |index|
63
+ result_pointer = (array_pointer + (index - 1)).read_pointer
64
+ new(result_pointer)
65
+ end
66
+ end
67
+ end
68
+
69
+ # Extracts attributes of rd_kafka_DeleteAcls_result_response_t
70
+ #
71
+ # @private
72
+ class DeleteAclResult
73
+ attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
74
+
75
+ def initialize(acl_result_pointer)
76
+ @matching_acls=[]
77
+ rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_error(acl_result_pointer)
78
+ @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
79
+ @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
80
+ if @result_error == 0
81
+ # Get the number of matching acls
82
+ pointer_to_size_t = FFI::MemoryPointer.new(:int32)
83
+ @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
84
+ @matching_acls_count = pointer_to_size_t.read_int
85
+ end
86
+ end
87
+
88
+ def self.delete_acl_results_from_array(count, array_pointer)
89
+ (1..count).map do |index|
90
+ result_pointer = (array_pointer + (index - 1)).read_pointer
91
+ new(result_pointer)
92
+ end
93
+ end
94
+ end
95
+
96
+ # Extracts attributes of rd_kafka_DeleteAcls_result_response_t
97
+ #
98
+ # @private
99
+ class DescribeAclResult
100
+ attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
101
+
102
+ def initialize(event_ptr)
103
+ @matching_acls=[]
104
+ @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
105
+ @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
106
+ if @result_error == 0
107
+ acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
108
+ # Get the number of matching acls
109
+ pointer_to_size_t = FFI::MemoryPointer.new(:int32)
110
+ @matching_acls = Rdkafka::Bindings.rd_kafka_DescribeAcls_result_acls(acl_describe_result, pointer_to_size_t)
111
+ @matching_acls_count = pointer_to_size_t.read_int
112
+ end
113
+ end
114
+ end
115
+
24
116
  # FFI Function used for Create Topic and Delete Topic callbacks
25
117
  BackgroundEventCallbackFunction = FFI::Function.new(
26
118
  :void, [:pointer, :pointer, :pointer]
@@ -36,6 +128,16 @@ module Rdkafka
36
128
  process_create_topic(event_ptr)
37
129
  elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
38
130
  process_delete_topic(event_ptr)
131
+ elsif event_type == Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
132
+ process_create_partitions(event_ptr)
133
+ elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
134
+ process_create_acl(event_ptr)
135
+ elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
136
+ process_delete_acl(event_ptr)
137
+ elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
138
+ process_describe_acl(event_ptr)
139
+ elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
140
+ process_delete_groups(event_ptr)
39
141
  end
40
142
  end
41
143
 
@@ -58,6 +160,23 @@ module Rdkafka
58
160
  end
59
161
  end
60
162
 
163
+ def self.process_delete_groups(event_ptr)
164
+ delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
165
+
166
+ # Get the number of delete group results
167
+ pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
168
+ delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
169
+ delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array) # TODO fix this
170
+ delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
171
+
172
+ if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
173
+ delete_group_handle[:response] = delete_group_results[0].result_error
174
+ delete_group_handle[:error_string] = delete_group_results[0].error_string
175
+ delete_group_handle[:result_name] = delete_group_results[0].result_name
176
+ delete_group_handle[:pending] = false
177
+ end
178
+ end
179
+
61
180
  def self.process_delete_topic(event_ptr)
62
181
  delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
63
182
 
@@ -74,6 +193,75 @@ module Rdkafka
74
193
  delete_topic_handle[:pending] = false
75
194
  end
76
195
  end
196
+
197
+ def self.process_create_partitions(event_ptr)
198
+ create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
199
+
200
+ # Get the number of create topic results
201
+ pointer_to_size_t = FFI::MemoryPointer.new(:int32)
202
+ create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
203
+ create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
204
+ create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
205
+
206
+ if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
207
+ create_partitions_handle[:response] = create_partitions_results[0].result_error
208
+ create_partitions_handle[:error_string] = create_partitions_results[0].error_string
209
+ create_partitions_handle[:result_name] = create_partitions_results[0].result_name
210
+ create_partitions_handle[:pending] = false
211
+ end
212
+ end
213
+
214
+ def self.process_create_acl(event_ptr)
215
+ create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
216
+
217
+ # Get the number of acl results
218
+ pointer_to_size_t = FFI::MemoryPointer.new(:int32)
219
+ create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
220
+ create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
221
+ create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
222
+
223
+ if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
224
+ create_acl_handle[:response] = create_acl_results[0].result_error
225
+ create_acl_handle[:response_string] = create_acl_results[0].error_string
226
+ create_acl_handle[:pending] = false
227
+ end
228
+ end
229
+
230
+ def self.process_delete_acl(event_ptr)
231
+ delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
232
+
233
+ # Get the number of acl results
234
+ pointer_to_size_t = FFI::MemoryPointer.new(:int32)
235
+ delete_acl_result_responses = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_responses(delete_acls_result, pointer_to_size_t)
236
+ delete_acl_results = DeleteAclResult.delete_acl_results_from_array(pointer_to_size_t.read_int, delete_acl_result_responses)
237
+ delete_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
238
+
239
+ if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
240
+ delete_acl_handle[:response] = delete_acl_results[0].result_error
241
+ delete_acl_handle[:response_string] = delete_acl_results[0].error_string
242
+ delete_acl_handle[:pending] = false
243
+ if delete_acl_results[0].result_error == 0
244
+ delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
245
+ delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
246
+ end
247
+ end
248
+ end
249
+
250
+ def self.process_describe_acl(event_ptr)
251
+ describe_acl = DescribeAclResult.new(event_ptr)
252
+ describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
253
+
254
+ if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
255
+ describe_acl_handle[:response] = describe_acl.result_error
256
+ describe_acl_handle[:response_string] = describe_acl.error_string
257
+ describe_acl_handle[:pending] = false
258
+ if describe_acl.result_error == 0
259
+ describe_acl_handle[:acls] = describe_acl.matching_acls
260
+ describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
261
+ end
262
+ end
263
+ end
264
+
77
265
  end
78
266
 
79
267
  # FFI Function used for Message Delivery callbacks
@@ -90,17 +278,22 @@ module Rdkafka
90
278
  message = Rdkafka::Bindings::Message.new(message_ptr)
91
279
  delivery_handle_ptr_address = message[:_private].address
92
280
  if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
281
+ topic_name = Rdkafka::Bindings.rd_kafka_topic_name(message[:rkt])
282
+
93
283
  # Update delivery handle
94
284
  delivery_handle[:response] = message[:err]
95
285
  delivery_handle[:partition] = message[:partition]
96
286
  delivery_handle[:offset] = message[:offset]
287
+ delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
97
288
  delivery_handle[:pending] = false
289
+
98
290
  # Call delivery callback on opaque
99
291
  if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
100
- opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]), delivery_handle)
292
+ opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
101
293
  end
102
294
  end
103
295
  end
104
296
  end
297
+
105
298
  end
106
299
  end