rdkafka 0.8.0.beta.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.semaphore/semaphore.yml +23 -0
  3. data/CHANGELOG.md +18 -0
  4. data/README.md +5 -2
  5. data/docker-compose.yml +2 -0
  6. data/ext/README.md +7 -0
  7. data/ext/Rakefile +2 -1
  8. data/lib/rdkafka/abstract_handle.rb +82 -0
  9. data/lib/rdkafka/admin/create_topic_handle.rb +27 -0
  10. data/lib/rdkafka/admin/create_topic_report.rb +22 -0
  11. data/lib/rdkafka/admin/delete_topic_handle.rb +27 -0
  12. data/lib/rdkafka/admin/delete_topic_report.rb +22 -0
  13. data/lib/rdkafka/admin.rb +155 -0
  14. data/lib/rdkafka/bindings.rb +57 -18
  15. data/lib/rdkafka/callbacks.rb +106 -0
  16. data/lib/rdkafka/config.rb +59 -3
  17. data/lib/rdkafka/consumer.rb +125 -5
  18. data/lib/rdkafka/error.rb +29 -3
  19. data/lib/rdkafka/metadata.rb +6 -5
  20. data/lib/rdkafka/producer/delivery_handle.rb +7 -53
  21. data/lib/rdkafka/producer.rb +25 -11
  22. data/lib/rdkafka/version.rb +3 -3
  23. data/lib/rdkafka.rb +7 -0
  24. data/spec/rdkafka/abstract_handle_spec.rb +114 -0
  25. data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
  26. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  27. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
  28. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  29. data/spec/rdkafka/admin_spec.rb +203 -0
  30. data/spec/rdkafka/bindings_spec.rb +32 -8
  31. data/spec/rdkafka/callbacks_spec.rb +20 -0
  32. data/spec/rdkafka/config_spec.rb +76 -7
  33. data/spec/rdkafka/consumer_spec.rb +266 -2
  34. data/spec/rdkafka/error_spec.rb +4 -0
  35. data/spec/rdkafka/metadata_spec.rb +78 -0
  36. data/spec/rdkafka/producer/delivery_handle_spec.rb +1 -41
  37. data/spec/rdkafka/producer_spec.rb +98 -31
  38. data/spec/spec_helper.rb +28 -11
  39. metadata +32 -9
  40. data/.travis.yml +0 -45
@@ -0,0 +1,16 @@
1
+ require "spec_helper"
2
+
3
+ describe Rdkafka::Admin::CreateTopicReport do
4
+ subject { Rdkafka::Admin::CreateTopicReport.new(
5
+ FFI::MemoryPointer.from_string("error string"),
6
+ FFI::MemoryPointer.from_string("result name")
7
+ )}
8
+
9
+ it "should get the error string" do
10
+ expect(subject.error_string).to eq("error string")
11
+ end
12
+
13
+ it "should get the result name" do
14
+ expect(subject.result_name).to eq("result name")
15
+ end
16
+ end
@@ -0,0 +1,52 @@
1
+ require "spec_helper"
2
+
3
+ describe Rdkafka::Admin::DeleteTopicHandle do
4
+ let(:response) { 0 }
5
+
6
+ subject do
7
+ Rdkafka::Admin::DeleteTopicHandle.new.tap do |handle|
8
+ handle[:pending] = pending_handle
9
+ handle[:response] = response
10
+ handle[:error_string] = FFI::Pointer::NULL
11
+ handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
12
+ end
13
+ end
14
+
15
+ describe "#wait" do
16
+ let(:pending_handle) { true }
17
+
18
+ it "should wait until the timeout and then raise an error" do
19
+ expect {
20
+ subject.wait(max_wait_timeout: 0.1)
21
+ }.to raise_error Rdkafka::Admin::DeleteTopicHandle::WaitTimeoutError, /delete topic/
22
+ end
23
+
24
+ context "when not pending anymore and no error" do
25
+ let(:pending_handle) { false }
26
+
27
+ it "should return a delete topic report" do
28
+ report = subject.wait
29
+
30
+ expect(report.error_string).to eq(nil)
31
+ expect(report.result_name).to eq("my-test-topic")
32
+ end
33
+
34
+ it "should wait without a timeout" do
35
+ report = subject.wait(max_wait_timeout: nil)
36
+
37
+ expect(report.error_string).to eq(nil)
38
+ expect(report.result_name).to eq("my-test-topic")
39
+ end
40
+ end
41
+ end
42
+
43
+ describe "#raise_error" do
44
+ let(:pending_handle) { false }
45
+
46
+ it "should raise the appropriate error" do
47
+ expect {
48
+ subject.raise_error
49
+ }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,16 @@
1
+ require "spec_helper"
2
+
3
+ describe Rdkafka::Admin::DeleteTopicReport do
4
+ subject { Rdkafka::Admin::DeleteTopicReport.new(
5
+ FFI::MemoryPointer.from_string("error string"),
6
+ FFI::MemoryPointer.from_string("result name")
7
+ )}
8
+
9
+ it "should get the error string" do
10
+ expect(subject.error_string).to eq("error string")
11
+ end
12
+
13
+ it "should get the result name" do
14
+ expect(subject.result_name).to eq("result name")
15
+ end
16
+ end
@@ -0,0 +1,203 @@
1
+ require "spec_helper"
2
+ require "ostruct"
3
+
4
+ describe Rdkafka::Admin do
5
+ let(:config) { rdkafka_config }
6
+ let(:admin) { config.admin }
7
+
8
+ after do
9
+ # Registry should always end up being empty
10
+ expect(Rdkafka::Admin::CreateTopicHandle::REGISTRY).to be_empty
11
+ admin.close
12
+ end
13
+
14
+ let(:topic_name) { "test-topic-#{Random.new.rand(0..1_000_000)}" }
15
+ let(:topic_partition_count) { 3 }
16
+ let(:topic_replication_factor) { 1 }
17
+ let(:topic_config) { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
18
+ let(:invalid_topic_config) { {"cleeeeenup.policee" => "campact"} }
19
+
20
+ describe "#create_topic" do
21
+ describe "called with invalid input" do
22
+ describe "with an invalid topic name" do
23
+ # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
24
+ # public static final String LEGAL_CHARS = "[a-zA-Z0-9._-]";
25
+ let(:topic_name) { "[!@#]" }
26
+
27
+ it "raises an exception" do
28
+ create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
29
+ expect {
30
+ create_topic_handle.wait(max_wait_timeout: 15.0)
31
+ }.to raise_exception { |ex|
32
+ expect(ex).to be_a(Rdkafka::RdkafkaError)
33
+ expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
34
+ expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
35
+ }
36
+ end
37
+ end
38
+
39
+ describe "with the name of a topic that already exists" do
40
+ let(:topic_name) { "empty_test_topic" } # created in spec_helper.rb
41
+
42
+ it "raises an exception" do
43
+ create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
44
+ expect {
45
+ create_topic_handle.wait(max_wait_timeout: 15.0)
46
+ }.to raise_exception { |ex|
47
+ expect(ex).to be_a(Rdkafka::RdkafkaError)
48
+ expect(ex.message).to match(/Broker: Topic already exists \(topic_already_exists\)/)
49
+ expect(ex.broker_message).to match(/Topic 'empty_test_topic' already exists/)
50
+ }
51
+ end
52
+ end
53
+
54
+ describe "with an invalid partition count" do
55
+ let(:topic_partition_count) { -1 }
56
+
57
+ it "raises an exception" do
58
+ expect {
59
+ admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
60
+ }.to raise_error Rdkafka::Config::ConfigError, /num_partitions out of expected range/
61
+ end
62
+ end
63
+
64
+ describe "with an invalid replication factor" do
65
+ let(:topic_replication_factor) { -2 }
66
+
67
+ it "raises an exception" do
68
+ expect {
69
+ admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
70
+ }.to raise_error Rdkafka::Config::ConfigError, /replication_factor out of expected range/
71
+ end
72
+ end
73
+
74
+ describe "with an invalid topic configuration" do
75
+ it "doesn't create the topic" do
76
+ create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, invalid_topic_config)
77
+ expect {
78
+ create_topic_handle.wait(max_wait_timeout: 15.0)
79
+ }.to raise_error Rdkafka::RdkafkaError, /Broker: Configuration is invalid \(invalid_config\)/
80
+ end
81
+ end
82
+ end
83
+
84
+ context "edge case" do
85
+ context "where we are unable to get the background queue" do
86
+ before do
87
+ allow(Rdkafka::Bindings).to receive(:rd_kafka_queue_get_background).and_return(FFI::Pointer::NULL)
88
+ end
89
+
90
+ it "raises an exception" do
91
+ expect {
92
+ admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
93
+ }.to raise_error Rdkafka::Config::ConfigError, /rd_kafka_queue_get_background was NULL/
94
+ end
95
+ end
96
+
97
+ context "where rd_kafka_CreateTopics raises an exception" do
98
+ before do
99
+ allow(Rdkafka::Bindings).to receive(:rd_kafka_CreateTopics).and_raise(RuntimeError.new("oops"))
100
+ end
101
+
102
+ it "raises an exception" do
103
+ expect {
104
+ admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
105
+ }.to raise_error RuntimeError, /oops/
106
+ end
107
+ end
108
+ end
109
+
110
+ it "creates a topic" do
111
+ create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, topic_config)
112
+ create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
113
+ expect(create_topic_report.error_string).to be_nil
114
+ expect(create_topic_report.result_name).to eq(topic_name)
115
+ end
116
+ end
117
+
118
+ describe "#delete_topic" do
119
+ describe "called with invalid input" do
120
+ # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
121
+ # public static final String LEGAL_CHARS = "[a-zA-Z0-9._-]";
122
+ describe "with an invalid topic name" do
123
+ let(:topic_name) { "[!@#]" }
124
+
125
+ it "raises an exception" do
126
+ delete_topic_handle = admin.delete_topic(topic_name)
127
+ expect {
128
+ delete_topic_handle.wait(max_wait_timeout: 15.0)
129
+ }.to raise_exception { |ex|
130
+ expect(ex).to be_a(Rdkafka::RdkafkaError)
131
+ expect(ex.message).to match(/Broker: Unknown topic or partition \(unknown_topic_or_part\)/)
132
+ expect(ex.broker_message).to match(/Broker: Unknown topic or partition/)
133
+ }
134
+ end
135
+ end
136
+
137
+ describe "with the name of a topic that does not exist" do
138
+ it "raises an exception" do
139
+ delete_topic_handle = admin.delete_topic(topic_name)
140
+ expect {
141
+ delete_topic_handle.wait(max_wait_timeout: 15.0)
142
+ }.to raise_exception { |ex|
143
+ expect(ex).to be_a(Rdkafka::RdkafkaError)
144
+ expect(ex.message).to match(/Broker: Unknown topic or partition \(unknown_topic_or_part\)/)
145
+ expect(ex.broker_message).to match(/Broker: Unknown topic or partition/)
146
+ }
147
+ end
148
+ end
149
+ end
150
+
151
+ context "edge case" do
152
+ context "where we are unable to get the background queue" do
153
+ before do
154
+ allow(Rdkafka::Bindings).to receive(:rd_kafka_queue_get_background).and_return(FFI::Pointer::NULL)
155
+ end
156
+
157
+ it "raises an exception" do
158
+ expect {
159
+ admin.delete_topic(topic_name)
160
+ }.to raise_error Rdkafka::Config::ConfigError, /rd_kafka_queue_get_background was NULL/
161
+ end
162
+ end
163
+
164
+ context "where rd_kafka_DeleteTopics raises an exception" do
165
+ before do
166
+ allow(Rdkafka::Bindings).to receive(:rd_kafka_DeleteTopics).and_raise(RuntimeError.new("oops"))
167
+ end
168
+
169
+ it "raises an exception" do
170
+ expect {
171
+ admin.delete_topic(topic_name)
172
+ }.to raise_error RuntimeError, /oops/
173
+ end
174
+ end
175
+ end
176
+
177
+
178
+ it "deletes a topic that was newly created" do
179
+ create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
180
+ create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
181
+ expect(create_topic_report.error_string).to be_nil
182
+ expect(create_topic_report.result_name).to eq(topic_name)
183
+
184
+ # Retry topic deletion a few times. On CI Kafka seems to not
185
+ # always be ready for it immediately
186
+ delete_topic_report = nil
187
+ 10.times do |i|
188
+ begin
189
+ delete_topic_handle = admin.delete_topic(topic_name)
190
+ delete_topic_report = delete_topic_handle.wait(max_wait_timeout: 15.0)
191
+ break
192
+ rescue Rdkafka::RdkafkaError => ex
193
+ if i > 3
194
+ raise ex
195
+ end
196
+ end
197
+ end
198
+
199
+ expect(delete_topic_report.error_string).to be_nil
200
+ expect(delete_topic_report.result_name).to eq(topic_name)
201
+ end
202
+ end
203
+ end
@@ -25,39 +25,39 @@ describe Rdkafka::Bindings do
25
25
  end
26
26
 
27
27
  describe "log callback" do
28
- let(:log) { StringIO.new }
28
+ let(:log_queue) { Rdkafka::Config.log_queue }
29
29
  before do
30
- Rdkafka::Config.logger = Logger.new(log)
30
+ allow(log_queue).to receive(:<<)
31
31
  end
32
32
 
33
33
  it "should log fatal messages" do
34
34
  Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
35
- expect(log.string).to include "FATAL -- : rdkafka: log line"
35
+ expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
36
36
  end
37
37
 
38
38
  it "should log error messages" do
39
39
  Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
40
- expect(log.string).to include "ERROR -- : rdkafka: log line"
40
+ expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
41
41
  end
42
42
 
43
43
  it "should log warning messages" do
44
44
  Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
45
- expect(log.string).to include "WARN -- : rdkafka: log line"
45
+ expect(log_queue).to have_received(:<<).with([Logger::WARN, "rdkafka: log line"])
46
46
  end
47
47
 
48
48
  it "should log info messages" do
49
49
  Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
50
- expect(log.string).to include "INFO -- : rdkafka: log line"
50
+ expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
51
51
  end
52
52
 
53
53
  it "should log debug messages" do
54
54
  Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
55
- expect(log.string).to include "DEBUG -- : rdkafka: log line"
55
+ expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
56
56
  end
57
57
 
58
58
  it "should log unknown messages" do
59
59
  Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
60
- expect(log.string).to include "ANY -- : rdkafka: log line"
60
+ expect(log_queue).to have_received(:<<).with([Logger::UNKNOWN, "rdkafka: log line"])
61
61
  end
62
62
  end
63
63
 
@@ -100,4 +100,28 @@ describe Rdkafka::Bindings do
100
100
  end
101
101
  end
102
102
  end
103
+
104
+ describe "error callback" do
105
+ context "without an error callback" do
106
+ it "should do nothing" do
107
+ expect {
108
+ Rdkafka::Bindings::ErrorCallback.call(nil, 1, "error", nil)
109
+ }.not_to raise_error
110
+ end
111
+ end
112
+
113
+ context "with an error callback" do
114
+ before do
115
+ Rdkafka::Config.error_callback = lambda do |error|
116
+ $received_error = error
117
+ end
118
+ end
119
+
120
+ it "should call the error callback with an Rdkafka::Error" do
121
+ Rdkafka::Bindings::ErrorCallback.call(nil, 8, "Broker not available", nil)
122
+ expect($received_error.code).to eq(:broker_not_available)
123
+ expect($received_error.broker_message).to eq("Broker not available")
124
+ end
125
+ end
126
+ end
103
127
  end
@@ -0,0 +1,20 @@
1
+ require "spec_helper"
2
+
3
+ describe Rdkafka::Callbacks do
4
+
5
+ # The code in the call back functions is 100% covered by other specs. Due to
6
+ # the large number of collaborators, and the fact that FFI does not play
7
+ # nicely with doubles, it was very difficult to construct tests that were
8
+ # not over-mocked.
9
+
10
+ # For debugging purposes, if you suspect that you are running into trouble in
11
+ # one of the callback functions, it may be helpful to surround the inner body
12
+ # of the method with something like:
13
+ #
14
+ # begin
15
+ # <method body>
16
+ # rescue => ex; puts ex.inspect; puts ex.backtrace; end;
17
+ #
18
+ # This will output to STDOUT any exceptions that are being raised in the callback.
19
+
20
+ end
@@ -18,25 +18,83 @@ describe Rdkafka::Config do
18
18
  Rdkafka::Config.logger = nil
19
19
  }.to raise_error(Rdkafka::Config::NoLoggerError)
20
20
  end
21
+
22
+ it "supports logging queue" do
23
+ log = StringIO.new
24
+ Rdkafka::Config.logger = Logger.new(log)
25
+
26
+ Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
27
+ 20.times do
28
+ break if log.string != ""
29
+ sleep 0.05
30
+ end
31
+
32
+ expect(log.string).to include "FATAL -- : I love testing"
33
+ end
21
34
  end
22
35
 
23
36
  context "statistics callback" do
24
- it "should set the callback" do
25
- expect {
26
- Rdkafka::Config.statistics_callback = lambda do |stats|
27
- puts stats
37
+ context "with a proc/lambda" do
38
+ it "should set the callback" do
39
+ expect {
40
+ Rdkafka::Config.statistics_callback = lambda do |stats|
41
+ puts stats
42
+ end
43
+ }.not_to raise_error
44
+ expect(Rdkafka::Config.statistics_callback).to respond_to :call
45
+ end
46
+ end
47
+
48
+ context "with a callable object" do
49
+ it "should set the callback" do
50
+ callback = Class.new do
51
+ def call(stats); end
28
52
  end
29
- }.not_to raise_error
30
- expect(Rdkafka::Config.statistics_callback).to be_a Proc
53
+ expect {
54
+ Rdkafka::Config.statistics_callback = callback.new
55
+ }.not_to raise_error
56
+ expect(Rdkafka::Config.statistics_callback).to respond_to :call
57
+ end
31
58
  end
32
59
 
33
- it "should not accept a callback that's not a proc" do
60
+ it "should not accept a callback that's not callable" do
34
61
  expect {
35
62
  Rdkafka::Config.statistics_callback = 'a string'
36
63
  }.to raise_error(TypeError)
37
64
  end
38
65
  end
39
66
 
67
+ context "error callback" do
68
+ context "with a proc/lambda" do
69
+ it "should set the callback" do
70
+ expect {
71
+ Rdkafka::Config.error_callback = lambda do |error|
72
+ puts error
73
+ end
74
+ }.not_to raise_error
75
+ expect(Rdkafka::Config.error_callback).to respond_to :call
76
+ end
77
+ end
78
+
79
+ context "with a callable object" do
80
+ it "should set the callback" do
81
+ callback = Class.new do
82
+ def call(stats); end
83
+ end
84
+ expect {
85
+ Rdkafka::Config.error_callback = callback.new
86
+ }.not_to raise_error
87
+ expect(Rdkafka::Config.error_callback).to respond_to :call
88
+ end
89
+ end
90
+
91
+ it "should not accept a callback that's not callable" do
92
+ expect {
93
+ Rdkafka::Config.error_callback = 'a string'
94
+ }.to raise_error(TypeError)
95
+ end
96
+ end
97
+
40
98
  context "configuration" do
41
99
  it "should store configuration" do
42
100
  config = Rdkafka::Config.new
@@ -90,6 +148,17 @@ describe Rdkafka::Config do
90
148
  }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
91
149
  end
92
150
 
151
+ it "should allow configuring zstd compression" do
152
+ config = Rdkafka::Config.new('compression.codec' => 'zstd')
153
+ begin
154
+ expect(config.producer).to be_a Rdkafka::Producer
155
+ config.producer.close
156
+ rescue Rdkafka::Config::ConfigError => ex
157
+ pending "Zstd compression not supported on this machine"
158
+ raise ex
159
+ end
160
+ end
161
+
93
162
  it "should raise an error when client creation fails for a consumer" do
94
163
  config = Rdkafka::Config.new(
95
164
  "security.protocol" => "SSL",