rdkafka 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +10 -0
- data/CHANGELOG.md +5 -0
- data/README.md +4 -1
- data/Rakefile +48 -0
- data/lib/rdkafka/config.rb +1 -1
- data/lib/rdkafka/consumer.rb +33 -1
- data/lib/rdkafka/consumer/topic_partition_list.rb +7 -1
- data/lib/rdkafka/version.rb +2 -2
- data/rdkafka.gemspec +1 -0
- data/spec/rdkafka/bindings_spec.rb +49 -0
- data/spec/rdkafka/config_spec.rb +20 -0
- data/spec/rdkafka/consumer/message_spec.rb +1 -7
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +26 -1
- data/spec/rdkafka/consumer_spec.rb +198 -4
- data/spec/rdkafka/producer/delivery_handle_spec.rb +7 -0
- data/spec/rdkafka/producer_spec.rb +77 -22
- data/spec/spec_helper.rb +16 -2
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e665ee94b4d29ec2c4d618540ed4588a6af3cfe8
|
4
|
+
data.tar.gz: 7ad4f0ca3651861441bbe6009a503105e2c7f8c7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 712032c6802290695e4a4be376829c76da6d346b9af8d365e218f3edad511931526553625e7acdf588d1b71779bac2bcbcab43ac0b599ca1be3f592d0aac9759
|
7
|
+
data.tar.gz: 60c30f707e2112aaa63603429208a8fb98778dd5c08d3b29f8c700e6646755fbf1803d1258cf9701ba985636f2eb0eb2eed9bf623dbf174ac85bd2a3fc1bc21a
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -2,6 +2,10 @@ language: ruby
|
|
2
2
|
|
3
3
|
sudo: false
|
4
4
|
|
5
|
+
env:
|
6
|
+
global:
|
7
|
+
- CC_TEST_REPORTER_ID=0af2487f1cc190d4a5e23cd76182514e5b62aac5d59b6208a02fd518487b9ed8
|
8
|
+
|
5
9
|
rvm:
|
6
10
|
- 2.1
|
7
11
|
- 2.2
|
@@ -17,6 +21,12 @@ before_install:
|
|
17
21
|
before_script:
|
18
22
|
- cd ext && bundle exec rake && cd ..
|
19
23
|
- bundle exec rake create_topics
|
24
|
+
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
25
|
+
- chmod +x ./cc-test-reporter
|
26
|
+
- ./cc-test-reporter before-build
|
20
27
|
|
21
28
|
script:
|
22
29
|
- bundle exec rspec
|
30
|
+
|
31
|
+
after_script:
|
32
|
+
- ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
[](https://travis-ci.org/thijsc/rdkafka-ruby)
|
4
4
|
[](https://badge.fury.io/rb/rdkafka)
|
5
|
+
[](https://codeclimate.com/github/thijsc/rdkafka-ruby/maintainability)
|
6
|
+
[](https://codeclimate.com/github/thijsc/rdkafka-ruby/test_coverage)
|
5
7
|
|
6
8
|
The `rdkafka` gem is a modern Kafka client library for Ruby based on
|
7
9
|
[librdkafka](https://github.com/edenhill/librdkafka/).
|
@@ -54,7 +56,8 @@ end
|
|
54
56
|
|
55
57
|
## Development
|
56
58
|
|
57
|
-
|
59
|
+
For development we expect a local zookeeper and kafka instance to be
|
60
|
+
running. Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
|
58
61
|
create the topics as expected in the specs: `bundle exec rake create_topics`.
|
59
62
|
|
60
63
|
You can then run `bundle exec rspec` to run the tests. To see rdkafka
|
data/Rakefile
CHANGED
@@ -8,6 +8,8 @@ task :create_topics do
|
|
8
8
|
'kafka-topics'
|
9
9
|
end
|
10
10
|
`#{kafka_topics} --create --topic=consume_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
|
11
|
+
`#{kafka_topics} --create --topic=empty_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
|
12
|
+
`#{kafka_topics} --create --topic=load_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
|
11
13
|
`#{kafka_topics} --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
|
12
14
|
`#{kafka_topics} --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
|
13
15
|
end
|
@@ -44,3 +46,49 @@ task :consume_messages do
|
|
44
46
|
puts "Message received: #{message}"
|
45
47
|
end
|
46
48
|
end
|
49
|
+
|
50
|
+
task :load_test do
|
51
|
+
puts "Starting load test"
|
52
|
+
|
53
|
+
config = Rdkafka::Config.new(
|
54
|
+
:"bootstrap.servers" => "localhost:9092",
|
55
|
+
:"group.id" => "load-test",
|
56
|
+
:"enable.partition.eof" => false
|
57
|
+
)
|
58
|
+
|
59
|
+
# Create a producer in a thread
|
60
|
+
Thread.new do
|
61
|
+
producer = config.producer
|
62
|
+
loop do
|
63
|
+
handles = []
|
64
|
+
1000.times do |i|
|
65
|
+
handles.push(producer.produce(
|
66
|
+
topic: "load_test_topic",
|
67
|
+
payload: "Payload #{i}",
|
68
|
+
key: "Key #{i}"
|
69
|
+
))
|
70
|
+
end
|
71
|
+
handles.each(&:wait)
|
72
|
+
puts "Produced 1000 messages"
|
73
|
+
end
|
74
|
+
end.abort_on_exception = true
|
75
|
+
|
76
|
+
# Create three consumers in threads
|
77
|
+
3.times do |i|
|
78
|
+
Thread.new do
|
79
|
+
count = 0
|
80
|
+
consumer = config.consumer
|
81
|
+
consumer.subscribe("load_test_topic")
|
82
|
+
consumer.each do |message|
|
83
|
+
count += 1
|
84
|
+
if count % 1000 == 0
|
85
|
+
puts "Received 1000 messages in thread #{i}"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end.abort_on_exception = true
|
89
|
+
end
|
90
|
+
|
91
|
+
loop do
|
92
|
+
sleep 1
|
93
|
+
end
|
94
|
+
end
|
data/lib/rdkafka/config.rb
CHANGED
data/lib/rdkafka/consumer.rb
CHANGED
@@ -75,6 +75,8 @@ module Rdkafka
|
|
75
75
|
# Return the current committed offset per partition for this consumer group.
|
76
76
|
# The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
|
77
77
|
#
|
78
|
+
# TODO: This should use the subscription or assignment by default.
|
79
|
+
#
|
78
80
|
# @param list [TopicPartitionList] The topic with partitions to get the offsets for.
|
79
81
|
# @param timeout_ms [Integer] The timeout for fetching this information.
|
80
82
|
#
|
@@ -118,7 +120,37 @@ module Rdkafka
|
|
118
120
|
raise Rdkafka::RdkafkaError.new(response)
|
119
121
|
end
|
120
122
|
|
121
|
-
return low.
|
123
|
+
return low.read_int64, high.read_int64
|
124
|
+
end
|
125
|
+
|
126
|
+
# Calculate the consumer lag per partition for the provided topic partition list.
|
127
|
+
# You can get a suitable list by calling {committed} or {position} (TODO). It is also
|
128
|
+
# possible to create one yourself, in this case you have to provide a list that
|
129
|
+
# already contains all the partitions you need the lag for.
|
130
|
+
#
|
131
|
+
# @param topic_partition_list [TopicPartitionList] The list to calculate lag for.
|
132
|
+
# @param watermark_timeout_ms [Integer] The timeout for each query watermark call.
|
133
|
+
#
|
134
|
+
# @raise [RdkafkaError] When querying the broker fails.
|
135
|
+
#
|
136
|
+
# @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
|
137
|
+
def lag(topic_partition_list, watermark_timeout_ms=100)
|
138
|
+
out = {}
|
139
|
+
topic_partition_list.to_h.each do |topic, partitions|
|
140
|
+
# Query high watermarks for this topic's partitions
|
141
|
+
# and compare to the offset in the list.
|
142
|
+
topic_out = {}
|
143
|
+
partitions.each do |p|
|
144
|
+
low, high = query_watermark_offsets(
|
145
|
+
topic,
|
146
|
+
p.partition,
|
147
|
+
watermark_timeout_ms
|
148
|
+
)
|
149
|
+
topic_out[p.partition] = high - p.offset
|
150
|
+
end
|
151
|
+
out[topic] = topic_out
|
152
|
+
end
|
153
|
+
out
|
122
154
|
end
|
123
155
|
|
124
156
|
# Commit the current offsets of this consumer
|
@@ -31,8 +31,14 @@ module Rdkafka
|
|
31
31
|
|
32
32
|
# Add a topic with optionally partitions to the list.
|
33
33
|
#
|
34
|
+
# @example Add a topic with unassigned partitions
|
35
|
+
# tpl.add_topic("topic")
|
36
|
+
#
|
37
|
+
# @example Add a topic with assigned partitions
|
38
|
+
# tpl.add_topic("topic", (0..8))
|
39
|
+
#
|
34
40
|
# @param topic [String] The topic's name
|
35
|
-
# @param partition [Array<Integer>] The topic's partition's
|
41
|
+
# @param partition [Array<Integer>, Range<Integer>] The topic's partition's
|
36
42
|
#
|
37
43
|
# @return [nil]
|
38
44
|
def add_topic(topic, partitions=nil)
|
data/lib/rdkafka/version.rb
CHANGED
data/rdkafka.gemspec
CHANGED
@@ -5,9 +5,58 @@ describe Rdkafka::Bindings do
|
|
5
5
|
expect(Rdkafka::Bindings.ffi_libraries.map(&:name).first).to include "librdkafka"
|
6
6
|
end
|
7
7
|
|
8
|
+
describe ".lib_extension" do
|
9
|
+
it "should know the lib extension for darwin" do
|
10
|
+
expect(Gem::Platform.local).to receive(:os).and_return("darwin-aaa")
|
11
|
+
expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should know the lib extension for linux" do
|
15
|
+
expect(Gem::Platform.local).to receive(:os).and_return("linux")
|
16
|
+
expect(Rdkafka::Bindings.lib_extension).to eq "so"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
8
20
|
it "should successfully call librdkafka" do
|
9
21
|
expect {
|
10
22
|
Rdkafka::Bindings.rd_kafka_conf_new
|
11
23
|
}.not_to raise_error
|
12
24
|
end
|
25
|
+
|
26
|
+
describe "log callback" do
|
27
|
+
let(:log) { StringIO.new }
|
28
|
+
before do
|
29
|
+
Rdkafka::Config.logger = Logger.new(log)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should log fatal messages" do
|
33
|
+
Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
|
34
|
+
expect(log.string).to include "FATAL -- : rdkafka: log line"
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should log error messages" do
|
38
|
+
Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
|
39
|
+
expect(log.string).to include "ERROR -- : rdkafka: log line"
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should log warning messages" do
|
43
|
+
Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
|
44
|
+
expect(log.string).to include "WARN -- : rdkafka: log line"
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should log info messages" do
|
48
|
+
Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
|
49
|
+
expect(log.string).to include "INFO -- : rdkafka: log line"
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should log debug messages" do
|
53
|
+
Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
|
54
|
+
expect(log.string).to include "DEBUG -- : rdkafka: log line"
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should log unknown messages" do
|
58
|
+
Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
|
59
|
+
expect(log.string).to include "ANY -- : rdkafka: log line"
|
60
|
+
end
|
61
|
+
end
|
13
62
|
end
|
data/spec/rdkafka/config_spec.rb
CHANGED
@@ -53,5 +53,25 @@ describe Rdkafka::Config do
|
|
53
53
|
config.producer
|
54
54
|
}.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
|
55
55
|
end
|
56
|
+
|
57
|
+
it "should raise an error when client creation fails for a consumer" do
|
58
|
+
config = Rdkafka::Config.new(
|
59
|
+
"security.protocol" => "SSL",
|
60
|
+
"ssl.ca.location" => "/nonsense"
|
61
|
+
)
|
62
|
+
expect {
|
63
|
+
config.consumer
|
64
|
+
}.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should raise an error when client creation fails for a producer" do
|
68
|
+
config = Rdkafka::Config.new(
|
69
|
+
"security.protocol" => "SSL",
|
70
|
+
"ssl.ca.location" => "/nonsense"
|
71
|
+
)
|
72
|
+
expect {
|
73
|
+
config.producer
|
74
|
+
}.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
|
75
|
+
end
|
56
76
|
end
|
57
77
|
end
|
@@ -1,13 +1,7 @@
|
|
1
1
|
require "spec_helper"
|
2
2
|
|
3
3
|
describe Rdkafka::Consumer::Message do
|
4
|
-
let(:native_topic)
|
5
|
-
Rdkafka::Bindings.rd_kafka_topic_new(
|
6
|
-
native_client,
|
7
|
-
"topic_name",
|
8
|
-
nil
|
9
|
-
)
|
10
|
-
end
|
4
|
+
let(:native_topic) { new_native_topic }
|
11
5
|
let(:payload) { nil }
|
12
6
|
let(:key) { nil }
|
13
7
|
let(:native_message) do
|
@@ -37,7 +37,32 @@ describe Rdkafka::Consumer::TopicPartitionList do
|
|
37
37
|
})
|
38
38
|
end
|
39
39
|
|
40
|
-
it "should create a new list and add assigned topics" do
|
40
|
+
it "should create a new list and add assigned topics as a range" do
|
41
|
+
list = Rdkafka::Consumer::TopicPartitionList.new
|
42
|
+
|
43
|
+
expect(list.count).to eq 0
|
44
|
+
expect(list.empty?).to be true
|
45
|
+
|
46
|
+
list.add_topic("topic1", (0..2))
|
47
|
+
list.add_topic("topic2", (0..1))
|
48
|
+
|
49
|
+
expect(list.count).to eq 5
|
50
|
+
expect(list.empty?).to be false
|
51
|
+
|
52
|
+
hash = list.to_h
|
53
|
+
expect(hash.count).to eq 2
|
54
|
+
expect(hash["topic1"]).to eq([
|
55
|
+
Rdkafka::Consumer::Partition.new(0, -1001),
|
56
|
+
Rdkafka::Consumer::Partition.new(1, -1001),
|
57
|
+
Rdkafka::Consumer::Partition.new(2, -1001)
|
58
|
+
])
|
59
|
+
expect(hash["topic2"]).to eq([
|
60
|
+
Rdkafka::Consumer::Partition.new(0, -1001),
|
61
|
+
Rdkafka::Consumer::Partition.new(1, -1001)
|
62
|
+
])
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should create a new list and add assigned topics as an array" do
|
41
66
|
list = Rdkafka::Consumer::TopicPartitionList.new
|
42
67
|
|
43
68
|
expect(list.count).to eq 0
|
@@ -5,8 +5,8 @@ describe Rdkafka::Consumer do
|
|
5
5
|
let(:consumer) { config.consumer }
|
6
6
|
let(:producer) { config.producer }
|
7
7
|
|
8
|
-
|
9
|
-
it "should subscribe" do
|
8
|
+
describe "#subscripe, #unsubscribe and #subscription" do
|
9
|
+
it "should subscribe, unsubscribe and return the subscription" do
|
10
10
|
expect(consumer.subscription).to be_empty
|
11
11
|
|
12
12
|
consumer.subscribe("consume_test_topic")
|
@@ -21,9 +21,41 @@ describe Rdkafka::Consumer do
|
|
21
21
|
|
22
22
|
expect(consumer.subscription).to be_empty
|
23
23
|
end
|
24
|
+
|
25
|
+
it "should raise an error when subscribing fails" do
|
26
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_subscribe).and_return(20)
|
27
|
+
|
28
|
+
expect {
|
29
|
+
consumer.subscribe("consume_test_topic")
|
30
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should raise an error when unsubscribing fails" do
|
34
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_unsubscribe).and_return(20)
|
35
|
+
|
36
|
+
expect {
|
37
|
+
consumer.unsubscribe
|
38
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should raise an error when fetching the subscription fails" do
|
42
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_subscription).and_return(20)
|
43
|
+
|
44
|
+
expect {
|
45
|
+
consumer.subscription
|
46
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context "#close" do
|
51
|
+
it "should close a consumer" do
|
52
|
+
consumer.subscribe("consume_test_topic")
|
53
|
+
consumer.close
|
54
|
+
expect(consumer.poll(100)).to be_nil
|
55
|
+
end
|
24
56
|
end
|
25
57
|
|
26
|
-
describe "committed" do
|
58
|
+
describe "#commit and #committed" do
|
27
59
|
before do
|
28
60
|
# Make sure there's a stored offset
|
29
61
|
report = producer.produce(
|
@@ -32,6 +64,8 @@ describe Rdkafka::Consumer do
|
|
32
64
|
key: "key 1",
|
33
65
|
partition: 0
|
34
66
|
).wait
|
67
|
+
# Wait for message commits the current state,
|
68
|
+
# commit is therefore tested here.
|
35
69
|
message = wait_for_message(
|
36
70
|
topic: "consume_test_topic",
|
37
71
|
delivery_report: report,
|
@@ -39,6 +73,20 @@ describe Rdkafka::Consumer do
|
|
39
73
|
)
|
40
74
|
end
|
41
75
|
|
76
|
+
it "should only accept a topic partition list" do
|
77
|
+
expect {
|
78
|
+
consumer.committed("list")
|
79
|
+
}.to raise_error TypeError
|
80
|
+
end
|
81
|
+
|
82
|
+
it "should raise an error when committing fails" do
|
83
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
|
84
|
+
|
85
|
+
expect {
|
86
|
+
consumer.commit
|
87
|
+
}.to raise_error(Rdkafka::RdkafkaError)
|
88
|
+
end
|
89
|
+
|
42
90
|
it "should fetch the committed offsets for a specified topic partition list" do
|
43
91
|
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
44
92
|
list.add_topic("consume_test_topic", [0, 1, 2])
|
@@ -48,9 +96,19 @@ describe Rdkafka::Consumer do
|
|
48
96
|
expect(partitions[1].offset).to eq -1001
|
49
97
|
expect(partitions[2].offset).to eq -1001
|
50
98
|
end
|
99
|
+
|
100
|
+
it "should raise an error when getting committed fails" do
|
101
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
|
102
|
+
list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
|
103
|
+
list.add_topic("consume_test_topic", [0, 1, 2])
|
104
|
+
end
|
105
|
+
expect {
|
106
|
+
consumer.committed(list)
|
107
|
+
}.to raise_error Rdkafka::RdkafkaError
|
108
|
+
end
|
51
109
|
end
|
52
110
|
|
53
|
-
describe "
|
111
|
+
describe "#query_watermark_offsets" do
|
54
112
|
it "should return the watermark offsets" do
|
55
113
|
# Make sure there's a message
|
56
114
|
producer.produce(
|
@@ -64,5 +122,141 @@ describe Rdkafka::Consumer do
|
|
64
122
|
expect(low).to eq 0
|
65
123
|
expect(high).to be > 0
|
66
124
|
end
|
125
|
+
|
126
|
+
it "should raise an error when querying offsets fails" do
|
127
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_query_watermark_offsets).and_return(20)
|
128
|
+
expect {
|
129
|
+
consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
|
130
|
+
}.to raise_error Rdkafka::RdkafkaError
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
describe "#lag" do
|
135
|
+
let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
|
136
|
+
|
137
|
+
it "should calculate the consumer lag" do
|
138
|
+
# Make sure there's a message in every partition and
|
139
|
+
# wait for the message to make sure everything is committed.
|
140
|
+
(0..2).each do |i|
|
141
|
+
report = producer.produce(
|
142
|
+
topic: "consume_test_topic",
|
143
|
+
key: "key lag #{i}",
|
144
|
+
partition: i
|
145
|
+
).wait
|
146
|
+
end
|
147
|
+
|
148
|
+
# Consume to the end
|
149
|
+
consumer.subscribe("consume_test_topic")
|
150
|
+
eof_count = 0
|
151
|
+
loop do
|
152
|
+
begin
|
153
|
+
consumer.poll(100)
|
154
|
+
rescue Rdkafka::RdkafkaError => error
|
155
|
+
if error.is_partition_eof?
|
156
|
+
eof_count += 1
|
157
|
+
end
|
158
|
+
break if eof_count == 3
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
# Commit
|
163
|
+
consumer.commit
|
164
|
+
|
165
|
+
# Create list to fetch lag for. TODO creating the list will not be necessary
|
166
|
+
# after committed uses the subscription.
|
167
|
+
list = consumer.committed(Rdkafka::Consumer::TopicPartitionList.new.tap do |l|
|
168
|
+
l.add_topic("consume_test_topic", (0..2))
|
169
|
+
end)
|
170
|
+
|
171
|
+
# Lag should be 0 now
|
172
|
+
lag = consumer.lag(list)
|
173
|
+
expected_lag = {
|
174
|
+
"consume_test_topic" => {
|
175
|
+
0 => 0,
|
176
|
+
1 => 0,
|
177
|
+
2 => 0
|
178
|
+
}
|
179
|
+
}
|
180
|
+
expect(lag).to eq(expected_lag)
|
181
|
+
|
182
|
+
# Produce message on every topic again
|
183
|
+
(0..2).each do |i|
|
184
|
+
report = producer.produce(
|
185
|
+
topic: "consume_test_topic",
|
186
|
+
key: "key lag #{i}",
|
187
|
+
partition: i
|
188
|
+
).wait
|
189
|
+
end
|
190
|
+
|
191
|
+
# Lag should be 1 now
|
192
|
+
lag = consumer.lag(list)
|
193
|
+
expected_lag = {
|
194
|
+
"consume_test_topic" => {
|
195
|
+
0 => 1,
|
196
|
+
1 => 1,
|
197
|
+
2 => 1
|
198
|
+
}
|
199
|
+
}
|
200
|
+
expect(lag).to eq(expected_lag)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
describe "#poll" do
|
205
|
+
it "should return nil if there is no subscription" do
|
206
|
+
expect(consumer.poll(1000)).to be_nil
|
207
|
+
end
|
208
|
+
|
209
|
+
it "should return nil if there are no messages" do
|
210
|
+
consumer.subscribe("empty_test_topic")
|
211
|
+
expect(consumer.poll(1000)).to be_nil
|
212
|
+
end
|
213
|
+
|
214
|
+
it "should return a message if there is one" do
|
215
|
+
producer.produce(
|
216
|
+
topic: "consume_test_topic",
|
217
|
+
payload: "payload 1",
|
218
|
+
key: "key 1"
|
219
|
+
).wait
|
220
|
+
|
221
|
+
consumer.subscribe("consume_test_topic")
|
222
|
+
message = consumer.poll(5000)
|
223
|
+
expect(message).to be_a Rdkafka::Consumer::Message
|
224
|
+
|
225
|
+
# Message content is tested in producer spec
|
226
|
+
end
|
227
|
+
|
228
|
+
it "should raise an error when polling fails" do
|
229
|
+
message = Rdkafka::Bindings::Message.new.tap do |message|
|
230
|
+
message[:err] = 20
|
231
|
+
end
|
232
|
+
message_pointer = message.to_ptr
|
233
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_consumer_poll).and_return(message_pointer)
|
234
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_message_destroy).with(message_pointer)
|
235
|
+
expect {
|
236
|
+
consumer.poll(100)
|
237
|
+
}.to raise_error Rdkafka::RdkafkaError
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
describe "#each" do
|
242
|
+
it "should yield messages" do
|
243
|
+
10.times do
|
244
|
+
producer.produce(
|
245
|
+
topic: "consume_test_topic",
|
246
|
+
payload: "payload 1",
|
247
|
+
key: "key 1",
|
248
|
+
partition: 0
|
249
|
+
).wait
|
250
|
+
end
|
251
|
+
|
252
|
+
consumer.subscribe("consume_test_topic")
|
253
|
+
count = 0
|
254
|
+
# Check the first 10 messages
|
255
|
+
consumer.each do |message|
|
256
|
+
expect(message).to be_a Rdkafka::Consumer::Message
|
257
|
+
count += 1
|
258
|
+
break if count == 10
|
259
|
+
end
|
260
|
+
end
|
67
261
|
end
|
68
262
|
end
|
@@ -47,6 +47,13 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
47
47
|
expect(report.partition).to eq(2)
|
48
48
|
expect(report.offset).to eq(100)
|
49
49
|
end
|
50
|
+
|
51
|
+
it "should wait without a timeout" do
|
52
|
+
report = subject.wait(nil)
|
53
|
+
|
54
|
+
expect(report.partition).to eq(2)
|
55
|
+
expect(report.offset).to eq(100)
|
56
|
+
end
|
50
57
|
end
|
51
58
|
|
52
59
|
context "when not pending anymore and there was an error" do
|
@@ -13,19 +13,22 @@ describe Rdkafka::Producer do
|
|
13
13
|
end
|
14
14
|
|
15
15
|
it "should produce a message" do
|
16
|
+
# Produce a message
|
16
17
|
handle = producer.produce(
|
17
18
|
topic: "produce_test_topic",
|
18
|
-
payload: "payload
|
19
|
-
key: "key
|
19
|
+
payload: "payload",
|
20
|
+
key: "key"
|
20
21
|
)
|
22
|
+
|
23
|
+
# Should be pending at first
|
21
24
|
expect(handle.pending?).to be true
|
22
25
|
|
23
26
|
# Check delivery handle and report
|
24
27
|
report = handle.wait(5)
|
25
28
|
expect(handle.pending?).to be false
|
26
29
|
expect(report).not_to be_nil
|
27
|
-
expect(report.partition).to eq
|
28
|
-
expect(report.offset).to be
|
30
|
+
expect(report.partition).to eq 1
|
31
|
+
expect(report.offset).to be >= 0
|
29
32
|
|
30
33
|
# Close producer
|
31
34
|
producer.close
|
@@ -35,28 +38,41 @@ describe Rdkafka::Producer do
|
|
35
38
|
topic: "produce_test_topic",
|
36
39
|
delivery_report: report
|
37
40
|
)
|
38
|
-
expect(message.partition).to eq
|
39
|
-
expect(message.payload).to eq "payload
|
40
|
-
expect(message.key).to eq "key
|
41
|
+
expect(message.partition).to eq 1
|
42
|
+
expect(message.payload).to eq "payload"
|
43
|
+
expect(message.key).to eq "key"
|
41
44
|
# Since api.version.request is on by default we will get
|
42
45
|
# the message creation timestamp if it's not set.
|
43
46
|
expect(message.timestamp).to be > 1505069891557
|
44
47
|
end
|
45
48
|
|
49
|
+
it "should produce a message with a specified partition" do
|
50
|
+
# Produce a message
|
51
|
+
handle = producer.produce(
|
52
|
+
topic: "produce_test_topic",
|
53
|
+
payload: "payload partition",
|
54
|
+
key: "key partition",
|
55
|
+
partition: 1
|
56
|
+
)
|
57
|
+
report = handle.wait(5)
|
58
|
+
|
59
|
+
# Consume message and verify it's content
|
60
|
+
message = wait_for_message(
|
61
|
+
topic: "produce_test_topic",
|
62
|
+
delivery_report: report
|
63
|
+
)
|
64
|
+
expect(message.partition).to eq 1
|
65
|
+
expect(message.key).to eq "key partition"
|
66
|
+
end
|
67
|
+
|
46
68
|
it "should produce a message with utf-8 encoding" do
|
47
69
|
handle = producer.produce(
|
48
70
|
topic: "produce_test_topic",
|
49
71
|
payload: "Τη γλώσσα μου έδωσαν ελληνική",
|
50
72
|
key: "key utf8"
|
51
73
|
)
|
52
|
-
expect(handle.pending?).to be true
|
53
|
-
|
54
|
-
# Check delivery handle and report
|
55
74
|
report = handle.wait(5)
|
56
75
|
|
57
|
-
# Close producer
|
58
|
-
producer.close
|
59
|
-
|
60
76
|
# Consume message and verify it's content
|
61
77
|
message = wait_for_message(
|
62
78
|
topic: "produce_test_topic",
|
@@ -71,18 +87,12 @@ describe Rdkafka::Producer do
|
|
71
87
|
it "should produce a message with a timestamp" do
|
72
88
|
handle = producer.produce(
|
73
89
|
topic: "produce_test_topic",
|
74
|
-
payload: "
|
90
|
+
payload: "payload timestamp",
|
75
91
|
key: "key timestamp",
|
76
92
|
timestamp: 1505069646000
|
77
93
|
)
|
78
|
-
expect(handle.pending?).to be true
|
79
|
-
|
80
|
-
# Check delivery handle and report
|
81
94
|
report = handle.wait(5)
|
82
95
|
|
83
|
-
# Close producer
|
84
|
-
producer.close
|
85
|
-
|
86
96
|
# Consume message and verify it's content
|
87
97
|
message = wait_for_message(
|
88
98
|
topic: "produce_test_topic",
|
@@ -94,11 +104,56 @@ describe Rdkafka::Producer do
|
|
94
104
|
expect(message.timestamp).to eq 1505069646000
|
95
105
|
end
|
96
106
|
|
107
|
+
it "should produce a message with nil key" do
|
108
|
+
handle = producer.produce(
|
109
|
+
topic: "produce_test_topic",
|
110
|
+
payload: "payload no key"
|
111
|
+
)
|
112
|
+
report = handle.wait(5)
|
113
|
+
|
114
|
+
# Consume message and verify it's content
|
115
|
+
message = wait_for_message(
|
116
|
+
topic: "produce_test_topic",
|
117
|
+
delivery_report: report
|
118
|
+
)
|
119
|
+
|
120
|
+
expect(message.key).to be_nil
|
121
|
+
expect(message.payload).to eq "payload no key"
|
122
|
+
end
|
123
|
+
|
124
|
+
it "should produce a message with nil payload" do
|
125
|
+
handle = producer.produce(
|
126
|
+
topic: "produce_test_topic",
|
127
|
+
key: "key no payload"
|
128
|
+
)
|
129
|
+
report = handle.wait(5)
|
130
|
+
|
131
|
+
# Consume message and verify it's content
|
132
|
+
message = wait_for_message(
|
133
|
+
topic: "produce_test_topic",
|
134
|
+
delivery_report: report
|
135
|
+
)
|
136
|
+
|
137
|
+
expect(message.key).to eq "key no payload"
|
138
|
+
expect(message.payload).to be_nil
|
139
|
+
end
|
140
|
+
|
141
|
+
it "should raise an error when producing fails" do
|
142
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_producev).and_return(20)
|
143
|
+
|
144
|
+
expect {
|
145
|
+
producer.produce(
|
146
|
+
topic: "produce_test_topic",
|
147
|
+
key: "key error"
|
148
|
+
)
|
149
|
+
}.to raise_error Rdkafka::RdkafkaError
|
150
|
+
end
|
151
|
+
|
97
152
|
it "should raise a timeout error when waiting too long" do
|
98
153
|
handle = producer.produce(
|
99
154
|
topic: "produce_test_topic",
|
100
|
-
payload: "payload
|
101
|
-
key: "key
|
155
|
+
payload: "payload timeout",
|
156
|
+
key: "key timeout"
|
102
157
|
)
|
103
158
|
expect {
|
104
159
|
handle.wait(0)
|
data/spec/spec_helper.rb
CHANGED
@@ -1,8 +1,13 @@
|
|
1
|
+
require "simplecov"
|
2
|
+
SimpleCov.start do
|
3
|
+
add_filter "/spec/"
|
4
|
+
end
|
5
|
+
|
1
6
|
require "pry"
|
2
7
|
require "rspec"
|
3
8
|
require "rdkafka"
|
4
9
|
|
5
|
-
def rdkafka_config
|
10
|
+
def rdkafka_config(config_overrides={})
|
6
11
|
config = {
|
7
12
|
:"bootstrap.servers" => "localhost:9092",
|
8
13
|
:"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
|
@@ -14,6 +19,7 @@ def rdkafka_config
|
|
14
19
|
elsif ENV["DEBUG_CONSUMER"]
|
15
20
|
config[:debug] = "cgrp,topic,fetch"
|
16
21
|
end
|
22
|
+
config.merge!(config_overrides)
|
17
23
|
Rdkafka::Config.new(config)
|
18
24
|
end
|
19
25
|
|
@@ -22,6 +28,14 @@ def native_client
|
|
22
28
|
config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
|
23
29
|
end
|
24
30
|
|
31
|
+
def new_native_topic(topic_name="topic_name")
|
32
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
33
|
+
native_client,
|
34
|
+
topic_name,
|
35
|
+
nil
|
36
|
+
)
|
37
|
+
end
|
38
|
+
|
25
39
|
def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: nil)
|
26
40
|
config = rdkafka_config if config.nil?
|
27
41
|
consumer = config.consumer
|
@@ -34,7 +48,7 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: n
|
|
34
48
|
message = consumer.poll(100)
|
35
49
|
if message &&
|
36
50
|
message.partition == delivery_report.partition &&
|
37
|
-
message.offset == delivery_report.offset
|
51
|
+
message.offset == delivery_report.offset
|
38
52
|
return message
|
39
53
|
end
|
40
54
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-10-
|
11
|
+
date: 2017-10-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '12.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: simplecov
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0.15'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.15'
|
83
97
|
description: Modern Kafka client library for Ruby based on librdkafka
|
84
98
|
email:
|
85
99
|
- thijs@appsignal.com
|