rdkafka 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cd80c3cdbb5bc790349df27bdef345b86e37d341
4
- data.tar.gz: 822bad9c78b508e28cc7030443841b622b4d6df2
3
+ metadata.gz: e665ee94b4d29ec2c4d618540ed4588a6af3cfe8
4
+ data.tar.gz: 7ad4f0ca3651861441bbe6009a503105e2c7f8c7
5
5
  SHA512:
6
- metadata.gz: 63f00a3e196e10359220321613b03335bcec067e342155edbe256ef8a04716b435d2444674839e694e423a08a91e240e2e7d0756fb79402a354b8c8ab49cd3d1
7
- data.tar.gz: 4e112ca823c2b500187aed86edf540f3eda4fbb45cee3b22ee908a1605696545cff2a77e2b56ba2d73500e185a3c1524992638e6ccb31d8f3c14d32cb783c1bf
6
+ metadata.gz: 712032c6802290695e4a4be376829c76da6d346b9af8d365e218f3edad511931526553625e7acdf588d1b71779bac2bcbcab43ac0b599ca1be3f592d0aac9759
7
+ data.tar.gz: 60c30f707e2112aaa63603429208a8fb98778dd5c08d3b29f8c700e6646755fbf1803d1258cf9701ba985636f2eb0eb2eed9bf623dbf174ac85bd2a3fc1bc21a
data/.gitignore CHANGED
@@ -5,3 +5,4 @@ ext/librdkafka.*
5
5
  *.gem
6
6
  .yardoc
7
7
  doc
8
+ coverage
@@ -2,6 +2,10 @@ language: ruby
2
2
 
3
3
  sudo: false
4
4
 
5
+ env:
6
+ global:
7
+ - CC_TEST_REPORTER_ID=0af2487f1cc190d4a5e23cd76182514e5b62aac5d59b6208a02fd518487b9ed8
8
+
5
9
  rvm:
6
10
  - 2.1
7
11
  - 2.2
@@ -17,6 +21,12 @@ before_install:
17
21
  before_script:
18
22
  - cd ext && bundle exec rake && cd ..
19
23
  - bundle exec rake create_topics
24
+ - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
25
+ - chmod +x ./cc-test-reporter
26
+ - ./cc-test-reporter before-build
20
27
 
21
28
  script:
22
29
  - bundle exec rspec
30
+
31
+ after_script:
32
+ - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
@@ -1,3 +1,8 @@
1
+ # 0.3.1
2
+ * Bump librdkafka to 0.11.1
3
+ * Officially support ranges in `add_topic` for topic partition list.
4
+ * Add consumer lag calculator
5
+
1
6
  # 0.3.0
2
7
  * Move both add topic methods to one `add_topic` in `TopicPartitionList`
3
8
  * Add committed offsets to consumer
data/README.md CHANGED
@@ -2,6 +2,8 @@
2
2
 
3
3
  [![Build Status](https://travis-ci.org/thijsc/rdkafka-ruby.svg?branch=master)](https://travis-ci.org/thijsc/rdkafka-ruby)
4
4
  [![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
5
+ [![Maintainability](https://api.codeclimate.com/v1/badges/362d93880c1b6b6f48c6/maintainability)](https://codeclimate.com/github/thijsc/rdkafka-ruby/maintainability)
6
+ [![Test Coverage](https://api.codeclimate.com/v1/badges/362d93880c1b6b6f48c6/test_coverage)](https://codeclimate.com/github/thijsc/rdkafka-ruby/test_coverage)
5
7
 
6
8
  The `rdkafka` gem is a modern Kafka client library for Ruby based on
7
9
  [librdkafka](https://github.com/edenhill/librdkafka/).
@@ -54,7 +56,8 @@ end
54
56
 
55
57
  ## Development
56
58
 
57
- Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
59
+ For development we expect a local zookeeper and kafka instance to be
60
+ running. Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
58
61
  create the topics as expected in the specs: `bundle exec rake create_topics`.
59
62
 
60
63
  You can then run `bundle exec rspec` to run the tests. To see rdkafka
data/Rakefile CHANGED
@@ -8,6 +8,8 @@ task :create_topics do
8
8
  'kafka-topics'
9
9
  end
10
10
  `#{kafka_topics} --create --topic=consume_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
11
+ `#{kafka_topics} --create --topic=empty_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
12
+ `#{kafka_topics} --create --topic=load_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
11
13
  `#{kafka_topics} --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
12
14
  `#{kafka_topics} --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
13
15
  end
@@ -44,3 +46,49 @@ task :consume_messages do
44
46
  puts "Message received: #{message}"
45
47
  end
46
48
  end
49
+
50
+ task :load_test do
51
+ puts "Starting load test"
52
+
53
+ config = Rdkafka::Config.new(
54
+ :"bootstrap.servers" => "localhost:9092",
55
+ :"group.id" => "load-test",
56
+ :"enable.partition.eof" => false
57
+ )
58
+
59
+ # Create a producer in a thread
60
+ Thread.new do
61
+ producer = config.producer
62
+ loop do
63
+ handles = []
64
+ 1000.times do |i|
65
+ handles.push(producer.produce(
66
+ topic: "load_test_topic",
67
+ payload: "Payload #{i}",
68
+ key: "Key #{i}"
69
+ ))
70
+ end
71
+ handles.each(&:wait)
72
+ puts "Produced 1000 messages"
73
+ end
74
+ end.abort_on_exception = true
75
+
76
+ # Create three consumers in threads
77
+ 3.times do |i|
78
+ Thread.new do
79
+ count = 0
80
+ consumer = config.consumer
81
+ consumer.subscribe("load_test_topic")
82
+ consumer.each do |message|
83
+ count += 1
84
+ if count % 1000 == 0
85
+ puts "Received 1000 messages in thread #{i}"
86
+ end
87
+ end
88
+ end.abort_on_exception = true
89
+ end
90
+
91
+ loop do
92
+ sleep 1
93
+ end
94
+ end
@@ -136,7 +136,7 @@ module Rdkafka
136
136
  256
137
137
  )
138
138
 
139
- if handle.nil?
139
+ if handle.null?
140
140
  raise ClientCreationError.new(error_buffer.read_string)
141
141
  end
142
142
 
@@ -75,6 +75,8 @@ module Rdkafka
75
75
  # Return the current committed offset per partition for this consumer group.
76
76
  # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
77
77
  #
78
+ # TODO: This should use the subscription or assignment by default.
79
+ #
78
80
  # @param list [TopicPartitionList] The topic with partitions to get the offsets for.
79
81
  # @param timeout_ms [Integer] The timeout for fetching this information.
80
82
  #
@@ -118,7 +120,37 @@ module Rdkafka
118
120
  raise Rdkafka::RdkafkaError.new(response)
119
121
  end
120
122
 
121
- return low.read_int, high.read_int
123
+ return low.read_int64, high.read_int64
124
+ end
125
+
126
+ # Calculate the consumer lag per partition for the provided topic partition list.
127
+ # You can get a suitable list by calling {committed} or {position} (TODO). It is also
128
+ # possible to create one yourself, in this case you have to provide a list that
129
+ # already contains all the partitions you need the lag for.
130
+ #
131
+ # @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
132
+ # @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
133
+ #
134
+ # @raise [RdkafkaError] When querying the broker fails.
135
+ #
136
+ # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
137
+ def lag(topic_partition_list, watermark_timeout_ms=100)
138
+ out = {}
139
+ topic_partition_list.to_h.each do |topic, partitions|
140
+ # Query high watermarks for this topic's partitions
141
+ # and compare to the offset in the list.
142
+ topic_out = {}
143
+ partitions.each do |p|
144
+ low, high = query_watermark_offsets(
145
+ topic,
146
+ p.partition,
147
+ watermark_timeout_ms
148
+ )
149
+ topic_out[p.partition] = high - p.offset
150
+ end
151
+ out[topic] = topic_out
152
+ end
153
+ out
122
154
  end
123
155
 
124
156
  # Commit the current offsets of this consumer
@@ -31,8 +31,14 @@ module Rdkafka
31
31
 
32
32
  # Add a topic with optionally partitions to the list.
33
33
  #
34
+ # @example Add a topic with unassigned partitions
35
+ # tpl.add_topic("topic")
36
+ #
37
+ # @example Add a topic with assigned partitions
38
+ # tpl.add_topic("topic", (0..8))
39
+ #
34
40
  # @param topic [String] The topic's name
35
- # @param partition [Array<Integer>] The topic's partition's
41
+ # @param partition [Array<Integer>, Range<Integer>] The topic's partition's
36
42
  #
37
43
  # @return [nil]
38
44
  def add_topic(topic, partitions=nil)
@@ -1,4 +1,4 @@
1
1
  module Rdkafka
2
- VERSION = "0.3.0"
3
- LIBRDKAFKA_VERSION = "0.11.0"
2
+ VERSION = "0.3.1"
3
+ LIBRDKAFKA_VERSION = "0.11.1"
4
4
  end
@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
23
23
  gem.add_development_dependency 'pry', '~> 0.10'
24
24
  gem.add_development_dependency 'rspec', '~> 3.5'
25
25
  gem.add_development_dependency 'rake', '~> 12.0'
26
+ gem.add_development_dependency 'simplecov', '~> 0.15'
26
27
  end
@@ -5,9 +5,58 @@ describe Rdkafka::Bindings do
5
5
  expect(Rdkafka::Bindings.ffi_libraries.map(&:name).first).to include "librdkafka"
6
6
  end
7
7
 
8
+ describe ".lib_extension" do
9
+ it "should know the lib extension for darwin" do
10
+ expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
11
+ expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
12
+ end
13
+
14
+ it "should know the lib extension for linux" do
15
+ expect(Gem::Platform.local).to receive(:os).and_return("linux")
16
+ expect(Rdkafka::Bindings.lib_extension).to eq "so"
17
+ end
18
+ end
19
+
8
20
  it "should successfully call librdkafka" do
9
21
  expect {
10
22
  Rdkafka::Bindings.rd_kafka_conf_new
11
23
  }.not_to raise_error
12
24
  end
25
+
26
+ describe "log callback" do
27
+ let(:log) { StringIO.new }
28
+ before do
29
+ Rdkafka::Config.logger = Logger.new(log)
30
+ end
31
+
32
+ it "should log fatal messages" do
33
+ Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
34
+ expect(log.string).to include "FATAL -- : rdkafka: log line"
35
+ end
36
+
37
+ it "should log error messages" do
38
+ Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
39
+ expect(log.string).to include "ERROR -- : rdkafka: log line"
40
+ end
41
+
42
+ it "should log warning messages" do
43
+ Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
44
+ expect(log.string).to include "WARN -- : rdkafka: log line"
45
+ end
46
+
47
+ it "should log info messages" do
48
+ Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
49
+ expect(log.string).to include "INFO -- : rdkafka: log line"
50
+ end
51
+
52
+ it "should log debug messages" do
53
+ Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
54
+ expect(log.string).to include "DEBUG -- : rdkafka: log line"
55
+ end
56
+
57
+ it "should log unknown messages" do
58
+ Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
59
+ expect(log.string).to include "ANY -- : rdkafka: log line"
60
+ end
61
+ end
13
62
  end
@@ -53,5 +53,25 @@ describe Rdkafka::Config do
53
53
  config.producer
54
54
  }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
55
55
  end
56
+
57
+ it "should raise an error when client creation fails for a consumer" do
58
+ config = Rdkafka::Config.new(
59
+ "security.protocol" => "SSL",
60
+ "ssl.ca.location" => "/nonsense"
61
+ )
62
+ expect {
63
+ config.consumer
64
+ }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
65
+ end
66
+
67
+ it "should raise an error when client creation fails for a producer" do
68
+ config = Rdkafka::Config.new(
69
+ "security.protocol" => "SSL",
70
+ "ssl.ca.location" => "/nonsense"
71
+ )
72
+ expect {
73
+ config.producer
74
+ }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
75
+ end
56
76
  end
57
77
  end
@@ -1,13 +1,7 @@
1
1
  require "spec_helper"
2
2
 
3
3
  describe Rdkafka::Consumer::Message do
4
- let(:native_topic) do
5
- Rdkafka::Bindings.rd_kafka_topic_new(
6
- native_client,
7
- "topic_name",
8
- nil
9
- )
10
- end
4
+ let(:native_topic) { new_native_topic }
11
5
  let(:payload) { nil }
12
6
  let(:key) { nil }
13
7
  let(:native_message) do
@@ -37,7 +37,32 @@ describe Rdkafka::Consumer::TopicPartitionList do
37
37
  })
38
38
  end
39
39
 
40
- it "should create a new list and add assigned topics" do
40
+ it "should create a new list and add assigned topics as a range" do
41
+ list = Rdkafka::Consumer::TopicPartitionList.new
42
+
43
+ expect(list.count).to eq 0
44
+ expect(list.empty?).to be true
45
+
46
+ list.add_topic("topic1", (0..2))
47
+ list.add_topic("topic2", (0..1))
48
+
49
+ expect(list.count).to eq 5
50
+ expect(list.empty?).to be false
51
+
52
+ hash = list.to_h
53
+ expect(hash.count).to eq 2
54
+ expect(hash["topic1"]).to eq([
55
+ Rdkafka::Consumer::Partition.new(0, -1001),
56
+ Rdkafka::Consumer::Partition.new(1, -1001),
57
+ Rdkafka::Consumer::Partition.new(2, -1001)
58
+ ])
59
+ expect(hash["topic2"]).to eq([
60
+ Rdkafka::Consumer::Partition.new(0, -1001),
61
+ Rdkafka::Consumer::Partition.new(1, -1001)
62
+ ])
63
+ end
64
+
65
+ it "should create a new list and add assigned topics as an array" do
41
66
  list = Rdkafka::Consumer::TopicPartitionList.new
42
67
 
43
68
  expect(list.count).to eq 0
@@ -5,8 +5,8 @@ describe Rdkafka::Consumer do
5
5
  let(:consumer) { config.consumer }
6
6
  let(:producer) { config.producer }
7
7
 
8
- context "subscription" do
9
- it "should subscribe" do
8
+ describe "#subscripe, #unsubscribe and #subscription" do
9
+ it "should subscribe, unsubscribe and return the subscription" do
10
10
  expect(consumer.subscription).to be_empty
11
11
 
12
12
  consumer.subscribe("consume_test_topic")
@@ -21,9 +21,41 @@ describe Rdkafka::Consumer do
21
21
 
22
22
  expect(consumer.subscription).to be_empty
23
23
  end
24
+
25
+ it "should raise an error when subscribing fails" do
26
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_subscribe).and_return(20)
27
+
28
+ expect {
29
+ consumer.subscribe("consume_test_topic")
30
+ }.to raise_error(Rdkafka::RdkafkaError)
31
+ end
32
+
33
+ it "should raise an error when unsubscribing fails" do
34
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_unsubscribe).and_return(20)
35
+
36
+ expect {
37
+ consumer.unsubscribe
38
+ }.to raise_error(Rdkafka::RdkafkaError)
39
+ end
40
+
41
+ it "should raise an error when fetching the subscription fails" do
42
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_subscription).and_return(20)
43
+
44
+ expect {
45
+ consumer.subscription
46
+ }.to raise_error(Rdkafka::RdkafkaError)
47
+ end
48
+ end
49
+
50
+ context "#close" do
51
+ it "should close a consumer" do
52
+ consumer.subscribe("consume_test_topic")
53
+ consumer.close
54
+ expect(consumer.poll(100)).to be_nil
55
+ end
24
56
  end
25
57
 
26
- describe "committed" do
58
+ describe "#commit and #committed" do
27
59
  before do
28
60
  # Make sure there's a stored offset
29
61
  report = producer.produce(
@@ -32,6 +64,8 @@ describe Rdkafka::Consumer do
32
64
  key: "key 1",
33
65
  partition: 0
34
66
  ).wait
67
+ # Wait for message commits the current state,
68
+ # commit is therefore tested here.
35
69
  message = wait_for_message(
36
70
  topic: "consume_test_topic",
37
71
  delivery_report: report,
@@ -39,6 +73,20 @@ describe Rdkafka::Consumer do
39
73
  )
40
74
  end
41
75
 
76
+ it "should only accept a topic partition list" do
77
+ expect {
78
+ consumer.committed("list")
79
+ }.to raise_error TypeError
80
+ end
81
+
82
+ it "should raise an error when committing fails" do
83
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
84
+
85
+ expect {
86
+ consumer.commit
87
+ }.to raise_error(Rdkafka::RdkafkaError)
88
+ end
89
+
42
90
  it "should fetch the committed offsets for a specified topic partition list" do
43
91
  list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
44
92
  list.add_topic("consume_test_topic", [0, 1, 2])
@@ -48,9 +96,19 @@ describe Rdkafka::Consumer do
48
96
  expect(partitions[1].offset).to eq -1001
49
97
  expect(partitions[2].offset).to eq -1001
50
98
  end
99
+
100
+ it "should raise an error when getting committed fails" do
101
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
102
+ list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
103
+ list.add_topic("consume_test_topic", [0, 1, 2])
104
+ end
105
+ expect {
106
+ consumer.committed(list)
107
+ }.to raise_error Rdkafka::RdkafkaError
108
+ end
51
109
  end
52
110
 
53
- describe "watermark offsets" do
111
+ describe "#query_watermark_offsets" do
54
112
  it "should return the watermark offsets" do
55
113
  # Make sure there's a message
56
114
  producer.produce(
@@ -64,5 +122,141 @@ describe Rdkafka::Consumer do
64
122
  expect(low).to eq 0
65
123
  expect(high).to be > 0
66
124
  end
125
+
126
+ it "should raise an error when querying offsets fails" do
127
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_query_watermark_offsets).and_return(20)
128
+ expect {
129
+ consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
130
+ }.to raise_error Rdkafka::RdkafkaError
131
+ end
132
+ end
133
+
134
+ describe "#lag" do
135
+ let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
136
+
137
+ it "should calculate the consumer lag" do
138
+ # Make sure there's a message in every partition and
139
+ # wait for the message to make sure everything is committed.
140
+ (0..2).each do |i|
141
+ report = producer.produce(
142
+ topic: "consume_test_topic",
143
+ key: "key lag #{i}",
144
+ partition: i
145
+ ).wait
146
+ end
147
+
148
+ # Consume to the end
149
+ consumer.subscribe("consume_test_topic")
150
+ eof_count = 0
151
+ loop do
152
+ begin
153
+ consumer.poll(100)
154
+ rescue Rdkafka::RdkafkaError => error
155
+ if error.is_partition_eof?
156
+ eof_count += 1
157
+ end
158
+ break if eof_count == 3
159
+ end
160
+ end
161
+
162
+ # Commit
163
+ consumer.commit
164
+
165
+ # Create list to fetch lag for. TODO creating the list will not be necessary
166
+ # after committed uses the subscription.
167
+ list = consumer.committed(Rdkafka::Consumer::TopicPartitionList.new.tap do |l|
168
+ l.add_topic("consume_test_topic", (0..2))
169
+ end)
170
+
171
+ # Lag should be 0 now
172
+ lag = consumer.lag(list)
173
+ expected_lag = {
174
+ "consume_test_topic" => {
175
+ 0 => 0,
176
+ 1 => 0,
177
+ 2 => 0
178
+ }
179
+ }
180
+ expect(lag).to eq(expected_lag)
181
+
182
+ # Produce message on every topic again
183
+ (0..2).each do |i|
184
+ report = producer.produce(
185
+ topic: "consume_test_topic",
186
+ key: "key lag #{i}",
187
+ partition: i
188
+ ).wait
189
+ end
190
+
191
+ # Lag should be 1 now
192
+ lag = consumer.lag(list)
193
+ expected_lag = {
194
+ "consume_test_topic" => {
195
+ 0 => 1,
196
+ 1 => 1,
197
+ 2 => 1
198
+ }
199
+ }
200
+ expect(lag).to eq(expected_lag)
201
+ end
202
+ end
203
+
204
+ describe "#poll" do
205
+ it "should return nil if there is no subscription" do
206
+ expect(consumer.poll(1000)).to be_nil
207
+ end
208
+
209
+ it "should return nil if there are no messages" do
210
+ consumer.subscribe("empty_test_topic")
211
+ expect(consumer.poll(1000)).to be_nil
212
+ end
213
+
214
+ it "should return a message if there is one" do
215
+ producer.produce(
216
+ topic: "consume_test_topic",
217
+ payload: "payload 1",
218
+ key: "key 1"
219
+ ).wait
220
+
221
+ consumer.subscribe("consume_test_topic")
222
+ message = consumer.poll(5000)
223
+ expect(message).to be_a Rdkafka::Consumer::Message
224
+
225
+ # Message content is tested in producer spec
226
+ end
227
+
228
+ it "should raise an error when polling fails" do
229
+ message = Rdkafka::Bindings::Message.new.tap do |message|
230
+ message[:err] = 20
231
+ end
232
+ message_pointer = message.to_ptr
233
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_consumer_poll).and_return(message_pointer)
234
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_message_destroy).with(message_pointer)
235
+ expect {
236
+ consumer.poll(100)
237
+ }.to raise_error Rdkafka::RdkafkaError
238
+ end
239
+ end
240
+
241
+ describe "#each" do
242
+ it "should yield messages" do
243
+ 10.times do
244
+ producer.produce(
245
+ topic: "consume_test_topic",
246
+ payload: "payload 1",
247
+ key: "key 1",
248
+ partition: 0
249
+ ).wait
250
+ end
251
+
252
+ consumer.subscribe("consume_test_topic")
253
+ count = 0
254
+ # Check the first 10 messages
255
+ consumer.each do |message|
256
+ expect(message).to be_a Rdkafka::Consumer::Message
257
+ count += 1
258
+ break if count == 10
259
+ end
260
+ end
67
261
  end
68
262
  end
@@ -47,6 +47,13 @@ describe Rdkafka::Producer::DeliveryHandle do
47
47
  expect(report.partition).to eq(2)
48
48
  expect(report.offset).to eq(100)
49
49
  end
50
+
51
+ it "should wait without a timeout" do
52
+ report = subject.wait(nil)
53
+
54
+ expect(report.partition).to eq(2)
55
+ expect(report.offset).to eq(100)
56
+ end
50
57
  end
51
58
 
52
59
  context "when not pending anymore and there was an error" do
@@ -13,19 +13,22 @@ describe Rdkafka::Producer do
13
13
  end
14
14
 
15
15
  it "should produce a message" do
16
+ # Produce a message
16
17
  handle = producer.produce(
17
18
  topic: "produce_test_topic",
18
- payload: "payload 1",
19
- key: "key 1"
19
+ payload: "payload",
20
+ key: "key"
20
21
  )
22
+
23
+ # Should be pending at first
21
24
  expect(handle.pending?).to be true
22
25
 
23
26
  # Check delivery handle and report
24
27
  report = handle.wait(5)
25
28
  expect(handle.pending?).to be false
26
29
  expect(report).not_to be_nil
27
- expect(report.partition).to eq 0
28
- expect(report.offset).to be > 0
30
+ expect(report.partition).to eq 1
31
+ expect(report.offset).to be >= 0
29
32
 
30
33
  # Close producer
31
34
  producer.close
@@ -35,28 +38,41 @@ describe Rdkafka::Producer do
35
38
  topic: "produce_test_topic",
36
39
  delivery_report: report
37
40
  )
38
- expect(message.partition).to eq 0
39
- expect(message.payload).to eq "payload 1"
40
- expect(message.key).to eq "key 1"
41
+ expect(message.partition).to eq 1
42
+ expect(message.payload).to eq "payload"
43
+ expect(message.key).to eq "key"
41
44
  # Since api.version.request is on by default we will get
42
45
  # the message creation timestamp if it's not set.
43
46
  expect(message.timestamp).to be > 1505069891557
44
47
  end
45
48
 
49
+ it "should produce a message with a specified partition" do
50
+ # Produce a message
51
+ handle = producer.produce(
52
+ topic: "produce_test_topic",
53
+ payload: "payload partition",
54
+ key: "key partition",
55
+ partition: 1
56
+ )
57
+ report = handle.wait(5)
58
+
59
+ # Consume message and verify it's content
60
+ message = wait_for_message(
61
+ topic: "produce_test_topic",
62
+ delivery_report: report
63
+ )
64
+ expect(message.partition).to eq 1
65
+ expect(message.key).to eq "key partition"
66
+ end
67
+
46
68
  it "should produce a message with utf-8 encoding" do
47
69
  handle = producer.produce(
48
70
  topic: "produce_test_topic",
49
71
  payload: "Τη γλώσσα μου έδωσαν ελληνική",
50
72
  key: "key utf8"
51
73
  )
52
- expect(handle.pending?).to be true
53
-
54
- # Check delivery handle and report
55
74
  report = handle.wait(5)
56
75
 
57
- # Close producer
58
- producer.close
59
-
60
76
  # Consume message and verify it's content
61
77
  message = wait_for_message(
62
78
  topic: "produce_test_topic",
@@ -71,18 +87,12 @@ describe Rdkafka::Producer do
71
87
  it "should produce a message with a timestamp" do
72
88
  handle = producer.produce(
73
89
  topic: "produce_test_topic",
74
- payload: "Payload timestamp",
90
+ payload: "payload timestamp",
75
91
  key: "key timestamp",
76
92
  timestamp: 1505069646000
77
93
  )
78
- expect(handle.pending?).to be true
79
-
80
- # Check delivery handle and report
81
94
  report = handle.wait(5)
82
95
 
83
- # Close producer
84
- producer.close
85
-
86
96
  # Consume message and verify it's content
87
97
  message = wait_for_message(
88
98
  topic: "produce_test_topic",
@@ -94,11 +104,56 @@ describe Rdkafka::Producer do
94
104
  expect(message.timestamp).to eq 1505069646000
95
105
  end
96
106
 
107
+ it "should produce a message with nil key" do
108
+ handle = producer.produce(
109
+ topic: "produce_test_topic",
110
+ payload: "payload no key"
111
+ )
112
+ report = handle.wait(5)
113
+
114
+ # Consume message and verify it's content
115
+ message = wait_for_message(
116
+ topic: "produce_test_topic",
117
+ delivery_report: report
118
+ )
119
+
120
+ expect(message.key).to be_nil
121
+ expect(message.payload).to eq "payload no key"
122
+ end
123
+
124
+ it "should produce a message with nil payload" do
125
+ handle = producer.produce(
126
+ topic: "produce_test_topic",
127
+ key: "key no payload"
128
+ )
129
+ report = handle.wait(5)
130
+
131
+ # Consume message and verify it's content
132
+ message = wait_for_message(
133
+ topic: "produce_test_topic",
134
+ delivery_report: report
135
+ )
136
+
137
+ expect(message.key).to eq "key no payload"
138
+ expect(message.payload).to be_nil
139
+ end
140
+
141
+ it "should raise an error when producing fails" do
142
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_producev).and_return(20)
143
+
144
+ expect {
145
+ producer.produce(
146
+ topic: "produce_test_topic",
147
+ key: "key error"
148
+ )
149
+ }.to raise_error Rdkafka::RdkafkaError
150
+ end
151
+
97
152
  it "should raise a timeout error when waiting too long" do
98
153
  handle = producer.produce(
99
154
  topic: "produce_test_topic",
100
- payload: "payload 1",
101
- key: "key 1"
155
+ payload: "payload timeout",
156
+ key: "key timeout"
102
157
  )
103
158
  expect {
104
159
  handle.wait(0)
@@ -1,8 +1,13 @@
1
+ require "simplecov"
2
+ SimpleCov.start do
3
+ add_filter "/spec/"
4
+ end
5
+
1
6
  require "pry"
2
7
  require "rspec"
3
8
  require "rdkafka"
4
9
 
5
- def rdkafka_config
10
+ def rdkafka_config(config_overrides={})
6
11
  config = {
7
12
  :"bootstrap.servers" => "localhost:9092",
8
13
  :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
@@ -14,6 +19,7 @@ def rdkafka_config
14
19
  elsif ENV["DEBUG_CONSUMER"]
15
20
  config[:debug] = "cgrp,topic,fetch"
16
21
  end
22
+ config.merge!(config_overrides)
17
23
  Rdkafka::Config.new(config)
18
24
  end
19
25
 
@@ -22,6 +28,14 @@ def native_client
22
28
  config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
23
29
  end
24
30
 
31
+ def new_native_topic(topic_name="topic_name")
32
+ Rdkafka::Bindings.rd_kafka_topic_new(
33
+ native_client,
34
+ topic_name,
35
+ nil
36
+ )
37
+ end
38
+
25
39
  def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: nil)
26
40
  config = rdkafka_config if config.nil?
27
41
  consumer = config.consumer
@@ -34,7 +48,7 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: n
34
48
  message = consumer.poll(100)
35
49
  if message &&
36
50
  message.partition == delivery_report.partition &&
37
- message.offset == delivery_report.offset - 1
51
+ message.offset == delivery_report.offset
38
52
  return message
39
53
  end
40
54
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rdkafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thijs Cadier
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-10-17 00:00:00.000000000 Z
11
+ date: 2017-10-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ffi
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '12.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: simplecov
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '0.15'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '0.15'
83
97
  description: Modern Kafka client library for Ruby based on librdkafka
84
98
  email:
85
99
  - thijs@appsignal.com