rdkafka 0.22.2 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +101 -3
  3. data/README.md +21 -14
  4. data/Rakefile +21 -21
  5. data/bin/verify_kafka_warnings +39 -0
  6. data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.2.tar.gz} +0 -0
  7. data/docker-compose-ssl.yml +35 -0
  8. data/docker-compose.yml +2 -2
  9. data/ext/Rakefile +27 -27
  10. data/lib/rdkafka/abstract_handle.rb +54 -7
  11. data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
  12. data/lib/rdkafka/admin/config_binding_result.rb +2 -2
  13. data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
  14. data/lib/rdkafka/admin/create_acl_handle.rb +10 -6
  15. data/lib/rdkafka/admin/create_acl_report.rb +3 -2
  16. data/lib/rdkafka/admin/create_partitions_handle.rb +8 -7
  17. data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
  18. data/lib/rdkafka/admin/create_topic_handle.rb +8 -7
  19. data/lib/rdkafka/admin/create_topic_report.rb +3 -0
  20. data/lib/rdkafka/admin/delete_acl_handle.rb +9 -8
  21. data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
  22. data/lib/rdkafka/admin/delete_groups_handle.rb +10 -7
  23. data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
  24. data/lib/rdkafka/admin/delete_topic_handle.rb +8 -7
  25. data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
  26. data/lib/rdkafka/admin/describe_acl_handle.rb +9 -8
  27. data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
  28. data/lib/rdkafka/admin/describe_configs_handle.rb +7 -10
  29. data/lib/rdkafka/admin/describe_configs_report.rb +8 -6
  30. data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -10
  31. data/lib/rdkafka/admin/incremental_alter_configs_report.rb +8 -6
  32. data/lib/rdkafka/admin/list_offsets_handle.rb +31 -0
  33. data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
  34. data/lib/rdkafka/admin.rb +357 -159
  35. data/lib/rdkafka/bindings.rb +205 -111
  36. data/lib/rdkafka/callbacks/base_handler.rb +62 -0
  37. data/lib/rdkafka/callbacks/create_acl_handler.rb +37 -0
  38. data/lib/rdkafka/callbacks/create_partitions_handler.rb +37 -0
  39. data/lib/rdkafka/callbacks/create_topic_handler.rb +37 -0
  40. data/lib/rdkafka/callbacks/delete_acl_handler.rb +42 -0
  41. data/lib/rdkafka/callbacks/delete_groups_handler.rb +37 -0
  42. data/lib/rdkafka/callbacks/delete_topic_handler.rb +37 -0
  43. data/lib/rdkafka/callbacks/describe_acl_handler.rb +35 -0
  44. data/lib/rdkafka/callbacks/describe_configs_handler.rb +42 -0
  45. data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb +42 -0
  46. data/lib/rdkafka/callbacks/list_offsets_handler.rb +42 -0
  47. data/lib/rdkafka/callbacks.rb +125 -216
  48. data/lib/rdkafka/config.rb +126 -66
  49. data/lib/rdkafka/consumer/headers.rb +22 -7
  50. data/lib/rdkafka/consumer/message.rb +12 -11
  51. data/lib/rdkafka/consumer/partition.rb +15 -4
  52. data/lib/rdkafka/consumer/topic_partition_list.rb +64 -38
  53. data/lib/rdkafka/consumer.rb +513 -59
  54. data/lib/rdkafka/defaults.rb +125 -0
  55. data/lib/rdkafka/error.rb +40 -14
  56. data/lib/rdkafka/helpers/metadata.rb +29 -0
  57. data/lib/rdkafka/helpers/oauth.rb +45 -13
  58. data/lib/rdkafka/helpers/time.rb +5 -0
  59. data/lib/rdkafka/metadata.rb +128 -37
  60. data/lib/rdkafka/native_kafka.rb +91 -8
  61. data/lib/rdkafka/producer/delivery_handle.rb +6 -6
  62. data/lib/rdkafka/producer/delivery_report.rb +10 -6
  63. data/lib/rdkafka/producer/partitions_count_cache.rb +51 -55
  64. data/lib/rdkafka/producer.rb +193 -95
  65. data/lib/rdkafka/version.rb +6 -3
  66. data/lib/rdkafka.rb +15 -0
  67. data/package-lock.json +331 -0
  68. data/package.json +9 -0
  69. data/rdkafka.gemspec +58 -36
  70. data/renovate.json +39 -23
  71. metadata +39 -125
  72. data/.github/CODEOWNERS +0 -3
  73. data/.github/FUNDING.yml +0 -1
  74. data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
  75. data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
  76. data/.github/workflows/ci_macos_arm64.yml +0 -284
  77. data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
  78. data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
  79. data/.github/workflows/push_macos_arm64.yml +0 -54
  80. data/.github/workflows/push_ruby.yml +0 -37
  81. data/.github/workflows/verify-action-pins.yml +0 -16
  82. data/.gitignore +0 -14
  83. data/.rspec +0 -2
  84. data/.ruby-gemset +0 -1
  85. data/.ruby-version +0 -1
  86. data/.yardopts +0 -2
  87. data/Gemfile +0 -5
  88. data/ext/README.md +0 -19
  89. data/ext/build_common.sh +0 -361
  90. data/ext/build_linux_x86_64_gnu.sh +0 -306
  91. data/ext/build_linux_x86_64_musl.sh +0 -763
  92. data/ext/build_macos_arm64.sh +0 -550
  93. data/spec/rdkafka/abstract_handle_spec.rb +0 -117
  94. data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
  95. data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
  96. data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
  97. data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
  98. data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
  99. data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
  100. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
  101. data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
  102. data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
  103. data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
  104. data/spec/rdkafka/admin_spec.rb +0 -971
  105. data/spec/rdkafka/bindings_spec.rb +0 -199
  106. data/spec/rdkafka/callbacks_spec.rb +0 -20
  107. data/spec/rdkafka/config_spec.rb +0 -258
  108. data/spec/rdkafka/consumer/headers_spec.rb +0 -73
  109. data/spec/rdkafka/consumer/message_spec.rb +0 -139
  110. data/spec/rdkafka/consumer/partition_spec.rb +0 -57
  111. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
  112. data/spec/rdkafka/consumer_spec.rb +0 -1274
  113. data/spec/rdkafka/error_spec.rb +0 -89
  114. data/spec/rdkafka/metadata_spec.rb +0 -79
  115. data/spec/rdkafka/native_kafka_spec.rb +0 -130
  116. data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
  117. data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
  118. data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
  119. data/spec/rdkafka/producer_spec.rb +0 -1345
  120. data/spec/spec_helper.rb +0 -195
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
+ # Callback handlers for librdkafka events
5
+ # @private
4
6
  module Callbacks
5
7
  # Extracts attributes of a rd_kafka_topic_result_t
6
8
  #
@@ -8,12 +10,16 @@ module Rdkafka
8
10
  class TopicResult
9
11
  attr_reader :result_error, :error_string, :result_name
10
12
 
13
+ # @param topic_result_pointer [FFI::Pointer] pointer to the topic result struct
11
14
  def initialize(topic_result_pointer)
12
15
  @result_error = Rdkafka::Bindings.rd_kafka_topic_result_error(topic_result_pointer)
13
16
  @error_string = Rdkafka::Bindings.rd_kafka_topic_result_error_string(topic_result_pointer)
14
17
  @result_name = Rdkafka::Bindings.rd_kafka_topic_result_name(topic_result_pointer)
15
18
  end
16
19
 
20
+ # @param count [Integer] number of results
21
+ # @param array_pointer [FFI::Pointer] pointer to the results array
22
+ # @return [Array<TopicResult>] array of topic results
17
23
  def self.create_topic_results_from_array(count, array_pointer)
18
24
  (1..count).map do |index|
19
25
  result_pointer = (array_pointer + (index - 1)).read_pointer
@@ -22,13 +28,18 @@ module Rdkafka
22
28
  end
23
29
  end
24
30
 
31
+ # Extracts attributes of rd_kafka_group_result_t
32
+ #
33
+ # @private
25
34
  class GroupResult
26
35
  attr_reader :result_error, :error_string, :result_name
36
+
37
+ # @param group_result_pointer [FFI::Pointer] pointer to the group result struct
27
38
  def initialize(group_result_pointer)
28
39
  native_error = Rdkafka::Bindings.rd_kafka_group_result_error(group_result_pointer)
29
40
 
30
41
  if native_error.null?
31
- @result_error = 0
42
+ @result_error = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
32
43
  @error_string = FFI::Pointer::NULL
33
44
  else
34
45
  @result_error = native_error[:code]
@@ -37,6 +48,10 @@ module Rdkafka
37
48
 
38
49
  @result_name = Rdkafka::Bindings.rd_kafka_group_result_name(group_result_pointer)
39
50
  end
51
+
52
+ # @param count [Integer] number of results
53
+ # @param array_pointer [FFI::Pointer] pointer to the results array
54
+ # @return [Array<GroupResult>] array of group results
40
55
  def self.create_group_results_from_array(count, array_pointer)
41
56
  (1..count).map do |index|
42
57
  result_pointer = (array_pointer + (index - 1)).read_pointer
@@ -51,12 +66,16 @@ module Rdkafka
51
66
  class CreateAclResult
52
67
  attr_reader :result_error, :error_string
53
68
 
69
+ # @param acl_result_pointer [FFI::Pointer] pointer to the ACL result struct
54
70
  def initialize(acl_result_pointer)
55
71
  rd_kafka_error_pointer = Bindings.rd_kafka_acl_result_error(acl_result_pointer)
56
72
  @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
57
73
  @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
58
74
  end
59
75
 
76
+ # @param count [Integer] number of results
77
+ # @param array_pointer [FFI::Pointer] pointer to the results array
78
+ # @return [Array<CreateAclResult>] array of ACL results
60
79
  def self.create_acl_results_from_array(count, array_pointer)
61
80
  (1..count).map do |index|
62
81
  result_pointer = (array_pointer + (index - 1)).read_pointer
@@ -71,19 +90,23 @@ module Rdkafka
71
90
  class DeleteAclResult
72
91
  attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
73
92
 
93
+ # @param acl_result_pointer [FFI::Pointer] pointer to the delete ACL result response struct
74
94
  def initialize(acl_result_pointer)
75
- @matching_acls=[]
95
+ @matching_acls = []
76
96
  rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_error(acl_result_pointer)
77
97
  @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
78
98
  @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
79
- if @result_error == 0
80
- # Get the number of matching acls
81
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
82
- @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
83
- @matching_acls_count = pointer_to_size_t.read_int
99
+ if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
100
+ # Get the number of matching acls
101
+ pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
102
+ @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
103
+ @matching_acls_count = pointer_to_size_t.read_int
84
104
  end
85
105
  end
86
106
 
107
+ # @param count [Integer] number of results
108
+ # @param array_pointer [FFI::Pointer] pointer to the results array
109
+ # @return [Array<DeleteAclResult>] array of delete ACL results
87
110
  def self.delete_acl_results_from_array(count, array_pointer)
88
111
  (1..count).map do |index|
89
112
  result_pointer = (array_pointer + (index - 1)).read_pointer
@@ -98,251 +121,123 @@ module Rdkafka
98
121
  class DescribeAclResult
99
122
  attr_reader :result_error, :error_string, :matching_acls, :matching_acls_count
100
123
 
124
+ # @param event_ptr [FFI::Pointer] pointer to the event
101
125
  def initialize(event_ptr)
102
- @matching_acls=[]
126
+ @matching_acls = []
103
127
  @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
104
128
  @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
105
- if @result_error == 0
129
+ if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
106
130
  acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
107
131
  # Get the number of matching acls
108
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
132
+ pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
109
133
  @matching_acls = Rdkafka::Bindings.rd_kafka_DescribeAcls_result_acls(acl_describe_result, pointer_to_size_t)
110
134
  @matching_acls_count = pointer_to_size_t.read_int
111
135
  end
112
136
  end
113
137
  end
114
138
 
139
+ # Extracts attributes of rd_kafka_DescribeConfigs_result_t
140
+ #
141
+ # @private
115
142
  class DescribeConfigsResult
116
143
  attr_reader :result_error, :error_string, :results, :results_count
117
144
 
145
+ # @param event_ptr [FFI::Pointer] pointer to the event
118
146
  def initialize(event_ptr)
119
- @results=[]
147
+ @results = []
120
148
  @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
121
149
  @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
122
150
 
123
- if @result_error == 0
151
+ if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
124
152
  configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
125
153
  # Get the number of matching acls
126
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
154
+ pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
127
155
  @results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
128
156
  @results_count = pointer_to_size_t.read_int
129
157
  end
130
158
  end
131
159
  end
132
160
 
161
+ # Extracts attributes of rd_kafka_IncrementalAlterConfigs_result_t
162
+ #
163
+ # @private
133
164
  class IncrementalAlterConfigsResult
134
165
  attr_reader :result_error, :error_string, :results, :results_count
135
166
 
167
+ # @param event_ptr [FFI::Pointer] pointer to the event
136
168
  def initialize(event_ptr)
137
- @results=[]
169
+ @results = []
138
170
  @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
139
171
  @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
140
172
 
141
- if @result_error == 0
173
+ if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
142
174
  incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
143
175
  # Get the number of matching acls
144
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
176
+ pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
145
177
  @results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
146
178
  @results_count = pointer_to_size_t.read_int
147
179
  end
148
180
  end
149
181
  end
150
182
 
183
+ # Extracts attributes of rd_kafka_ListOffsets_result_t
184
+ #
151
185
  # @private
152
- class BackgroundEventCallback
153
- def self.call(_, event_ptr, _)
154
- case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
155
- when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
156
- process_create_topic(event_ptr)
157
- when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
158
- process_describe_configs(event_ptr)
159
- when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
160
- process_incremental_alter_configs(event_ptr)
161
- when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
162
- process_delete_topic(event_ptr)
163
- when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
164
- process_create_partitions(event_ptr)
165
- when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
166
- process_create_acl(event_ptr)
167
- when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
168
- process_delete_acl(event_ptr)
169
- when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
170
- process_describe_acl(event_ptr)
171
- when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
172
- process_delete_groups(event_ptr)
173
- end
174
- end
175
-
176
- private
177
-
178
- def self.process_create_topic(event_ptr)
179
- create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
180
-
181
- # Get the number of create topic results
182
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
183
- create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
184
- create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
185
- create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
186
-
187
- if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
188
- create_topic_handle[:response] = create_topic_results[0].result_error
189
- create_topic_handle[:error_string] = create_topic_results[0].error_string
190
- create_topic_handle[:result_name] = create_topic_results[0].result_name
191
-
192
- create_topic_handle.unlock
193
- end
194
- end
195
-
196
- def self.process_describe_configs(event_ptr)
197
- describe_configs = DescribeConfigsResult.new(event_ptr)
198
- describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
199
-
200
- if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
201
- describe_configs_handle[:response] = describe_configs.result_error
202
- describe_configs_handle[:response_string] = describe_configs.error_string
203
- describe_configs_handle[:pending] = false
204
-
205
- if describe_configs.result_error == 0
206
- describe_configs_handle[:config_entries] = describe_configs.results
207
- describe_configs_handle[:entry_count] = describe_configs.results_count
208
- end
209
-
210
- describe_configs_handle.unlock
211
- end
212
- end
213
-
214
- def self.process_incremental_alter_configs(event_ptr)
215
- incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
216
- incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
217
-
218
- if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
219
- incremental_alter_handle[:response] = incremental_alter.result_error
220
- incremental_alter_handle[:response_string] = incremental_alter.error_string
221
- incremental_alter_handle[:pending] = false
222
-
223
- if incremental_alter.result_error == 0
224
- incremental_alter_handle[:config_entries] = incremental_alter.results
225
- incremental_alter_handle[:entry_count] = incremental_alter.results_count
226
- end
227
-
228
- incremental_alter_handle.unlock
229
- end
230
- end
231
-
232
- def self.process_delete_groups(event_ptr)
233
- delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
234
-
235
- # Get the number of delete group results
236
- pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
237
- delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
238
- delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array) # TODO fix this
239
- delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
240
-
241
- if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
242
- delete_group_handle[:response] = delete_group_results[0].result_error
243
- delete_group_handle[:error_string] = delete_group_results[0].error_string
244
- delete_group_handle[:result_name] = delete_group_results[0].result_name
186
+ class ListOffsetsResult
187
+ attr_reader :result_error, :error_string, :result_infos, :result_count
245
188
 
246
- delete_group_handle.unlock
247
- end
248
- end
249
-
250
- def self.process_delete_topic(event_ptr)
251
- delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
252
-
253
- # Get the number of topic results
254
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
255
- delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
256
- delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
257
- delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
258
-
259
- if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
260
- delete_topic_handle[:response] = delete_topic_results[0].result_error
261
- delete_topic_handle[:error_string] = delete_topic_results[0].error_string
262
- delete_topic_handle[:result_name] = delete_topic_results[0].result_name
263
-
264
- delete_topic_handle.unlock
265
- end
266
- end
267
-
268
- def self.process_create_partitions(event_ptr)
269
- create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
270
-
271
- # Get the number of create topic results
272
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
273
- create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
274
- create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
275
- create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
276
-
277
- if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
278
- create_partitions_handle[:response] = create_partitions_results[0].result_error
279
- create_partitions_handle[:error_string] = create_partitions_results[0].error_string
280
- create_partitions_handle[:result_name] = create_partitions_results[0].result_name
281
-
282
- create_partitions_handle.unlock
283
- end
284
- end
285
-
286
- def self.process_create_acl(event_ptr)
287
- create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
288
-
289
- # Get the number of acl results
290
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
291
- create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
292
- create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
293
- create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
294
-
295
- if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
296
- create_acl_handle[:response] = create_acl_results[0].result_error
297
- create_acl_handle[:response_string] = create_acl_results[0].error_string
189
+ # @param event_ptr [FFI::Pointer] pointer to the event
190
+ def initialize(event_ptr)
191
+ @result_infos = FFI::Pointer::NULL
192
+ @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
193
+ @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
298
194
 
299
- create_acl_handle.unlock
195
+ if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
196
+ list_offsets_result = Rdkafka::Bindings.rd_kafka_event_ListOffsets_result(event_ptr)
197
+ pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
198
+ @result_infos = Rdkafka::Bindings.rd_kafka_ListOffsets_result_infos(list_offsets_result, pointer_to_size_t)
199
+ @result_count = pointer_to_size_t.read(:size_t)
300
200
  end
301
201
  end
202
+ end
302
203
 
303
- def self.process_delete_acl(event_ptr)
304
- delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
305
-
306
- # Get the number of acl results
307
- pointer_to_size_t = FFI::MemoryPointer.new(:int32)
308
- delete_acl_result_responses = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_responses(delete_acls_result, pointer_to_size_t)
309
- delete_acl_results = DeleteAclResult.delete_acl_results_from_array(pointer_to_size_t.read_int, delete_acl_result_responses)
310
- delete_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
311
-
312
- if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
313
- delete_acl_handle[:response] = delete_acl_results[0].result_error
314
- delete_acl_handle[:response_string] = delete_acl_results[0].error_string
315
-
316
- if delete_acl_results[0].result_error == 0
317
- delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
318
- delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
319
- end
320
-
321
- delete_acl_handle.unlock
204
+ # @private
205
+ class BackgroundEventCallback
206
+ # Handles background events from librdkafka
207
+ # @param _client_ptr [FFI::Pointer] unused client pointer
208
+ # @param event_ptr [FFI::Pointer] pointer to the event
209
+ # @param _opaque_ptr [FFI::Pointer] unused opaque pointer
210
+ def self.call(_client_ptr, event_ptr, _opaque_ptr)
211
+ handler = case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
212
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT then CreateTopicHandler
213
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
214
+ when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
215
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
216
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
217
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
218
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT then DescribeAclHandler
219
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT then DescribeConfigsHandler
220
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
221
+ when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
322
222
  end
323
- end
324
-
325
- def self.process_describe_acl(event_ptr)
326
- describe_acl = DescribeAclResult.new(event_ptr)
327
- describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
328
-
329
- if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
330
- describe_acl_handle[:response] = describe_acl.result_error
331
- describe_acl_handle[:response_string] = describe_acl.error_string
332
-
333
- if describe_acl.result_error == 0
334
- describe_acl_handle[:acls] = describe_acl.matching_acls
335
- describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
336
- end
337
223
 
338
- describe_acl_handle.unlock
339
- end
224
+ handler&.call(event_ptr)
225
+ ensure
226
+ # Background queue events are owned by the application and must be destroyed once
227
+ # served (see rd_kafka_conf_set_background_event_cb). All event-owned memory (result
228
+ # arrays, strings) has to be copied into Ruby objects by the handlers above before this
229
+ # runs - nothing may retain pointers into the event past this point.
230
+ Rdkafka::Bindings.rd_kafka_event_destroy(event_ptr)
340
231
  end
341
232
  end
342
233
 
343
234
  # @private
344
235
  class DeliveryCallback
345
- def self.call(_, message_ptr, opaque_ptr)
236
+ # Handles message delivery callbacks
237
+ # @param _client_ptr [FFI::Pointer] unused client pointer
238
+ # @param message_ptr [FFI::Pointer] pointer to the delivered message
239
+ # @param opaque_ptr [FFI::Pointer] pointer to the opaque object for callback context
240
+ def self.call(_client_ptr, message_ptr, opaque_ptr)
346
241
  message = Rdkafka::Bindings::Message.new(message_ptr)
347
242
  delivery_handle_ptr_address = message[:_private].address
348
243
  if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
@@ -352,28 +247,42 @@ module Rdkafka
352
247
  delivery_handle[:response] = message[:err]
353
248
  delivery_handle[:partition] = message[:partition]
354
249
  delivery_handle[:offset] = message[:offset]
355
- delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
356
-
357
- # Call delivery callback on opaque
358
- if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
359
- opaque.call_delivery_callback(
360
- Rdkafka::Producer::DeliveryReport.new(
361
- message[:partition],
362
- message[:offset],
363
- topic_name,
364
- message[:err],
365
- delivery_handle.label
366
- ),
367
- delivery_handle
368
- )
250
+ # The topic is not stored on the handle here: it is already set as the handle's
251
+ # `topic` Ruby attribute during produce, which spares a native string copy that
252
+ # would otherwise be allocated and retained for every message.
253
+
254
+ begin
255
+ # Call delivery callback on opaque
256
+ if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
257
+ opaque.call_delivery_callback(
258
+ Rdkafka::Producer::DeliveryReport.new(
259
+ message[:partition],
260
+ message[:offset],
261
+ topic_name,
262
+ message[:err],
263
+ delivery_handle.label
264
+ ),
265
+ delivery_handle
266
+ )
267
+ end
268
+ rescue Exception => err
269
+ # A user delivery callback that raises must not escape this FFI callback: it runs on
270
+ # librdkafka's polling thread (abort_on_exception = true), so an escaping exception
271
+ # would crash the whole process. We log and swallow it, matching the rebalance
272
+ # callback. The `ensure` below still unlocks the handle, so `wait` returns the
273
+ # delivery report instead of blocking until its timeout and raising
274
+ # WaitTimeoutError for a message that was in fact delivered.
275
+ Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
276
+ ensure
277
+ delivery_handle.unlock
369
278
  end
370
-
371
- delivery_handle.unlock
372
279
  end
373
280
  end
374
281
  end
375
282
 
283
+ # @private
376
284
  @@mutex = Mutex.new
285
+ # @private
377
286
  @@current_pid = nil
378
287
 
379
288
  class << self
@@ -392,14 +301,14 @@ module Rdkafka
392
301
 
393
302
  # FFI Function used for Create Topic and Delete Topic callbacks
394
303
  background_event_callback_function = FFI::Function.new(
395
- :void, [:pointer, :pointer, :pointer]
304
+ :void, [:pointer, :pointer, :pointer]
396
305
  ) do |client_ptr, event_ptr, opaque_ptr|
397
306
  BackgroundEventCallback.call(client_ptr, event_ptr, opaque_ptr)
398
307
  end
399
308
 
400
309
  # FFI Function used for Message Delivery callbacks
401
310
  delivery_callback_function = FFI::Function.new(
402
- :void, [:pointer, :pointer, :pointer]
311
+ :void, [:pointer, :pointer, :pointer]
403
312
  ) do |client_ptr, message_ptr, opaque_ptr|
404
313
  DeliveryCallback.call(client_ptr, message_ptr, opaque_ptr)
405
314
  end