rdkafka 0.6.0 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.semaphore/semaphore.yml +23 -0
- data/CHANGELOG.md +27 -0
- data/README.md +9 -9
- data/docker-compose.yml +17 -11
- data/ext/README.md +10 -15
- data/ext/Rakefile +24 -3
- data/lib/rdkafka.rb +8 -0
- data/lib/rdkafka/abstract_handle.rb +82 -0
- data/lib/rdkafka/admin.rb +155 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +27 -0
- data/lib/rdkafka/admin/create_topic_report.rb +22 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +27 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +22 -0
- data/lib/rdkafka/bindings.rb +64 -18
- data/lib/rdkafka/callbacks.rb +106 -0
- data/lib/rdkafka/config.rb +38 -9
- data/lib/rdkafka/consumer.rb +221 -46
- data/lib/rdkafka/consumer/headers.rb +7 -5
- data/lib/rdkafka/consumer/partition.rb +1 -1
- data/lib/rdkafka/consumer/topic_partition_list.rb +6 -16
- data/lib/rdkafka/error.rb +35 -4
- data/lib/rdkafka/metadata.rb +92 -0
- data/lib/rdkafka/producer.rb +50 -24
- data/lib/rdkafka/producer/delivery_handle.rb +7 -49
- data/lib/rdkafka/producer/delivery_report.rb +7 -2
- data/lib/rdkafka/version.rb +3 -3
- data/rdkafka.gemspec +3 -3
- data/spec/rdkafka/abstract_handle_spec.rb +114 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin_spec.rb +203 -0
- data/spec/rdkafka/bindings_spec.rb +28 -10
- data/spec/rdkafka/callbacks_spec.rb +20 -0
- data/spec/rdkafka/config_spec.rb +51 -9
- data/spec/rdkafka/consumer/message_spec.rb +6 -1
- data/spec/rdkafka/consumer_spec.rb +287 -20
- data/spec/rdkafka/error_spec.rb +7 -3
- data/spec/rdkafka/metadata_spec.rb +78 -0
- data/spec/rdkafka/producer/delivery_handle_spec.rb +3 -43
- data/spec/rdkafka/producer/delivery_report_spec.rb +5 -1
- data/spec/rdkafka/producer_spec.rb +220 -100
- data/spec/spec_helper.rb +34 -6
- metadata +37 -13
- data/.travis.yml +0 -34
data/spec/spec_helper.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
unless ENV["CI"] == "true"
|
2
|
+
require "simplecov"
|
3
|
+
SimpleCov.start do
|
4
|
+
add_filter "/spec/"
|
5
|
+
end
|
4
6
|
end
|
5
7
|
|
6
8
|
require "pry"
|
@@ -25,12 +27,12 @@ def rdkafka_config(config_overrides={})
|
|
25
27
|
Rdkafka::Config.new(config)
|
26
28
|
end
|
27
29
|
|
28
|
-
def
|
30
|
+
def new_native_client
|
29
31
|
config = rdkafka_config
|
30
32
|
config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
|
31
33
|
end
|
32
34
|
|
33
|
-
def new_native_topic(topic_name="topic_name")
|
35
|
+
def new_native_topic(topic_name="topic_name", native_client: )
|
34
36
|
Rdkafka::Bindings.rd_kafka_topic_new(
|
35
37
|
native_client,
|
36
38
|
topic_name,
|
@@ -39,7 +41,8 @@ def new_native_topic(topic_name="topic_name")
|
|
39
41
|
end
|
40
42
|
|
41
43
|
def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
|
42
|
-
|
44
|
+
new_consumer = !!consumer
|
45
|
+
consumer ||= rdkafka_config.consumer
|
43
46
|
consumer.subscribe(topic)
|
44
47
|
timeout = Time.now.to_i + timeout_in_seconds
|
45
48
|
loop do
|
@@ -53,6 +56,8 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer:
|
|
53
56
|
return message
|
54
57
|
end
|
55
58
|
end
|
59
|
+
ensure
|
60
|
+
consumer.close if new_consumer
|
56
61
|
end
|
57
62
|
|
58
63
|
def wait_for_assignment(consumer)
|
@@ -68,3 +73,26 @@ def wait_for_unassignment(consumer)
|
|
68
73
|
sleep 1
|
69
74
|
end
|
70
75
|
end
|
76
|
+
|
77
|
+
RSpec.configure do |config|
|
78
|
+
config.before(:suite) do
|
79
|
+
admin = rdkafka_config.admin
|
80
|
+
{
|
81
|
+
consume_test_topic: 3,
|
82
|
+
empty_test_topic: 3,
|
83
|
+
load_test_topic: 3,
|
84
|
+
produce_test_topic: 3,
|
85
|
+
rake_test_topic: 3,
|
86
|
+
watermarks_test_topic: 3,
|
87
|
+
partitioner_test_topic: 25,
|
88
|
+
}.each do |topic, partitions|
|
89
|
+
create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
|
90
|
+
begin
|
91
|
+
create_topic_handle.wait(max_wait_timeout: 15)
|
92
|
+
rescue Rdkafka::RdkafkaError => ex
|
93
|
+
raise unless ex.message.match?(/topic_already_exists/)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
admin.close
|
97
|
+
end
|
98
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -42,14 +42,14 @@ dependencies:
|
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '12.3'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '12.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
@@ -117,7 +117,7 @@ extensions:
|
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
119
|
- ".gitignore"
|
120
|
-
- ".
|
120
|
+
- ".semaphore/semaphore.yml"
|
121
121
|
- ".yardopts"
|
122
122
|
- CHANGELOG.md
|
123
123
|
- Gemfile
|
@@ -128,7 +128,14 @@ files:
|
|
128
128
|
- ext/README.md
|
129
129
|
- ext/Rakefile
|
130
130
|
- lib/rdkafka.rb
|
131
|
+
- lib/rdkafka/abstract_handle.rb
|
132
|
+
- lib/rdkafka/admin.rb
|
133
|
+
- lib/rdkafka/admin/create_topic_handle.rb
|
134
|
+
- lib/rdkafka/admin/create_topic_report.rb
|
135
|
+
- lib/rdkafka/admin/delete_topic_handle.rb
|
136
|
+
- lib/rdkafka/admin/delete_topic_report.rb
|
131
137
|
- lib/rdkafka/bindings.rb
|
138
|
+
- lib/rdkafka/callbacks.rb
|
132
139
|
- lib/rdkafka/config.rb
|
133
140
|
- lib/rdkafka/consumer.rb
|
134
141
|
- lib/rdkafka/consumer/headers.rb
|
@@ -136,18 +143,27 @@ files:
|
|
136
143
|
- lib/rdkafka/consumer/partition.rb
|
137
144
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
138
145
|
- lib/rdkafka/error.rb
|
146
|
+
- lib/rdkafka/metadata.rb
|
139
147
|
- lib/rdkafka/producer.rb
|
140
148
|
- lib/rdkafka/producer/delivery_handle.rb
|
141
149
|
- lib/rdkafka/producer/delivery_report.rb
|
142
150
|
- lib/rdkafka/version.rb
|
143
151
|
- rdkafka.gemspec
|
152
|
+
- spec/rdkafka/abstract_handle_spec.rb
|
153
|
+
- spec/rdkafka/admin/create_topic_handle_spec.rb
|
154
|
+
- spec/rdkafka/admin/create_topic_report_spec.rb
|
155
|
+
- spec/rdkafka/admin/delete_topic_handle_spec.rb
|
156
|
+
- spec/rdkafka/admin/delete_topic_report_spec.rb
|
157
|
+
- spec/rdkafka/admin_spec.rb
|
144
158
|
- spec/rdkafka/bindings_spec.rb
|
159
|
+
- spec/rdkafka/callbacks_spec.rb
|
145
160
|
- spec/rdkafka/config_spec.rb
|
146
161
|
- spec/rdkafka/consumer/message_spec.rb
|
147
162
|
- spec/rdkafka/consumer/partition_spec.rb
|
148
163
|
- spec/rdkafka/consumer/topic_partition_list_spec.rb
|
149
164
|
- spec/rdkafka/consumer_spec.rb
|
150
165
|
- spec/rdkafka/error_spec.rb
|
166
|
+
- spec/rdkafka/metadata_spec.rb
|
151
167
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
152
168
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
153
169
|
- spec/rdkafka/producer_spec.rb
|
@@ -156,7 +172,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
|
|
156
172
|
licenses:
|
157
173
|
- MIT
|
158
174
|
metadata: {}
|
159
|
-
post_install_message:
|
175
|
+
post_install_message:
|
160
176
|
rdoc_options: []
|
161
177
|
require_paths:
|
162
178
|
- lib
|
@@ -164,27 +180,35 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
164
180
|
requirements:
|
165
181
|
- - ">="
|
166
182
|
- !ruby/object:Gem::Version
|
167
|
-
version: '2.
|
183
|
+
version: '2.4'
|
168
184
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
169
185
|
requirements:
|
170
186
|
- - ">="
|
171
187
|
- !ruby/object:Gem::Version
|
172
188
|
version: '0'
|
173
189
|
requirements: []
|
174
|
-
|
175
|
-
|
176
|
-
signing_key:
|
190
|
+
rubygems_version: 3.2.3
|
191
|
+
signing_key:
|
177
192
|
specification_version: 4
|
178
|
-
summary:
|
179
|
-
|
193
|
+
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
194
|
+
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|
195
|
+
and Ruby 2.4+.
|
180
196
|
test_files:
|
197
|
+
- spec/rdkafka/abstract_handle_spec.rb
|
198
|
+
- spec/rdkafka/admin/create_topic_handle_spec.rb
|
199
|
+
- spec/rdkafka/admin/create_topic_report_spec.rb
|
200
|
+
- spec/rdkafka/admin/delete_topic_handle_spec.rb
|
201
|
+
- spec/rdkafka/admin/delete_topic_report_spec.rb
|
202
|
+
- spec/rdkafka/admin_spec.rb
|
181
203
|
- spec/rdkafka/bindings_spec.rb
|
204
|
+
- spec/rdkafka/callbacks_spec.rb
|
182
205
|
- spec/rdkafka/config_spec.rb
|
183
206
|
- spec/rdkafka/consumer/message_spec.rb
|
184
207
|
- spec/rdkafka/consumer/partition_spec.rb
|
185
208
|
- spec/rdkafka/consumer/topic_partition_list_spec.rb
|
186
209
|
- spec/rdkafka/consumer_spec.rb
|
187
210
|
- spec/rdkafka/error_spec.rb
|
211
|
+
- spec/rdkafka/metadata_spec.rb
|
188
212
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
189
213
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
190
214
|
- spec/rdkafka/producer_spec.rb
|
data/.travis.yml
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
sudo: false
|
4
|
-
|
5
|
-
services:
|
6
|
-
- docker
|
7
|
-
|
8
|
-
env:
|
9
|
-
global:
|
10
|
-
- CC_TEST_REPORTER_ID=9f7f740ac1b6e264e1189fa07a6687a87bcdb9f3c0f4199d4344ab3b538e187e
|
11
|
-
- KAFKA_HEAP_OPTS="-Xmx512m -Xms512m"
|
12
|
-
|
13
|
-
rvm:
|
14
|
-
- 2.3
|
15
|
-
- 2.4
|
16
|
-
- 2.5
|
17
|
-
- 2.6
|
18
|
-
|
19
|
-
before_install:
|
20
|
-
- gem update --system
|
21
|
-
|
22
|
-
before_script:
|
23
|
-
- docker-compose up -d
|
24
|
-
- cd ext && bundle exec rake && cd ..
|
25
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
26
|
-
- chmod +x ./cc-test-reporter
|
27
|
-
- ./cc-test-reporter before-build
|
28
|
-
|
29
|
-
script:
|
30
|
-
- bundle exec rspec
|
31
|
-
|
32
|
-
after_script:
|
33
|
-
- docker-compose stop
|
34
|
-
- ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
|