rdkafka 0.6.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.semaphore/semaphore.yml +23 -0
- data/CHANGELOG.md +27 -0
- data/README.md +9 -9
- data/docker-compose.yml +17 -11
- data/ext/README.md +10 -15
- data/ext/Rakefile +24 -3
- data/lib/rdkafka.rb +8 -0
- data/lib/rdkafka/abstract_handle.rb +82 -0
- data/lib/rdkafka/admin.rb +155 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +27 -0
- data/lib/rdkafka/admin/create_topic_report.rb +22 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +27 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +22 -0
- data/lib/rdkafka/bindings.rb +64 -18
- data/lib/rdkafka/callbacks.rb +106 -0
- data/lib/rdkafka/config.rb +38 -9
- data/lib/rdkafka/consumer.rb +221 -46
- data/lib/rdkafka/consumer/headers.rb +7 -5
- data/lib/rdkafka/consumer/partition.rb +1 -1
- data/lib/rdkafka/consumer/topic_partition_list.rb +6 -16
- data/lib/rdkafka/error.rb +35 -4
- data/lib/rdkafka/metadata.rb +92 -0
- data/lib/rdkafka/producer.rb +50 -24
- data/lib/rdkafka/producer/delivery_handle.rb +7 -49
- data/lib/rdkafka/producer/delivery_report.rb +7 -2
- data/lib/rdkafka/version.rb +3 -3
- data/rdkafka.gemspec +3 -3
- data/spec/rdkafka/abstract_handle_spec.rb +114 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin_spec.rb +203 -0
- data/spec/rdkafka/bindings_spec.rb +28 -10
- data/spec/rdkafka/callbacks_spec.rb +20 -0
- data/spec/rdkafka/config_spec.rb +51 -9
- data/spec/rdkafka/consumer/message_spec.rb +6 -1
- data/spec/rdkafka/consumer_spec.rb +287 -20
- data/spec/rdkafka/error_spec.rb +7 -3
- data/spec/rdkafka/metadata_spec.rb +78 -0
- data/spec/rdkafka/producer/delivery_handle_spec.rb +3 -43
- data/spec/rdkafka/producer/delivery_report_spec.rb +5 -1
- data/spec/rdkafka/producer_spec.rb +220 -100
- data/spec/spec_helper.rb +34 -6
- metadata +37 -13
- data/.travis.yml +0 -34
data/spec/spec_helper.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
unless ENV["CI"] == "true"
|
2
|
+
require "simplecov"
|
3
|
+
SimpleCov.start do
|
4
|
+
add_filter "/spec/"
|
5
|
+
end
|
4
6
|
end
|
5
7
|
|
6
8
|
require "pry"
|
@@ -25,12 +27,12 @@ def rdkafka_config(config_overrides={})
|
|
25
27
|
Rdkafka::Config.new(config)
|
26
28
|
end
|
27
29
|
|
28
|
-
def
|
30
|
+
def new_native_client
|
29
31
|
config = rdkafka_config
|
30
32
|
config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
|
31
33
|
end
|
32
34
|
|
33
|
-
def new_native_topic(topic_name="topic_name")
|
35
|
+
def new_native_topic(topic_name="topic_name", native_client: )
|
34
36
|
Rdkafka::Bindings.rd_kafka_topic_new(
|
35
37
|
native_client,
|
36
38
|
topic_name,
|
@@ -39,7 +41,8 @@ def new_native_topic(topic_name="topic_name")
|
|
39
41
|
end
|
40
42
|
|
41
43
|
def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
|
42
|
-
|
44
|
+
new_consumer = !!consumer
|
45
|
+
consumer ||= rdkafka_config.consumer
|
43
46
|
consumer.subscribe(topic)
|
44
47
|
timeout = Time.now.to_i + timeout_in_seconds
|
45
48
|
loop do
|
@@ -53,6 +56,8 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer:
|
|
53
56
|
return message
|
54
57
|
end
|
55
58
|
end
|
59
|
+
ensure
|
60
|
+
consumer.close if new_consumer
|
56
61
|
end
|
57
62
|
|
58
63
|
def wait_for_assignment(consumer)
|
@@ -68,3 +73,26 @@ def wait_for_unassignment(consumer)
|
|
68
73
|
sleep 1
|
69
74
|
end
|
70
75
|
end
|
76
|
+
|
77
|
+
RSpec.configure do |config|
|
78
|
+
config.before(:suite) do
|
79
|
+
admin = rdkafka_config.admin
|
80
|
+
{
|
81
|
+
consume_test_topic: 3,
|
82
|
+
empty_test_topic: 3,
|
83
|
+
load_test_topic: 3,
|
84
|
+
produce_test_topic: 3,
|
85
|
+
rake_test_topic: 3,
|
86
|
+
watermarks_test_topic: 3,
|
87
|
+
partitioner_test_topic: 25,
|
88
|
+
}.each do |topic, partitions|
|
89
|
+
create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
|
90
|
+
begin
|
91
|
+
create_topic_handle.wait(max_wait_timeout: 15)
|
92
|
+
rescue Rdkafka::RdkafkaError => ex
|
93
|
+
raise unless ex.message.match?(/topic_already_exists/)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
admin.close
|
97
|
+
end
|
98
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -42,14 +42,14 @@ dependencies:
|
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '12.3'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '12.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
@@ -117,7 +117,7 @@ extensions:
|
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
119
|
- ".gitignore"
|
120
|
-
- ".
|
120
|
+
- ".semaphore/semaphore.yml"
|
121
121
|
- ".yardopts"
|
122
122
|
- CHANGELOG.md
|
123
123
|
- Gemfile
|
@@ -128,7 +128,14 @@ files:
|
|
128
128
|
- ext/README.md
|
129
129
|
- ext/Rakefile
|
130
130
|
- lib/rdkafka.rb
|
131
|
+
- lib/rdkafka/abstract_handle.rb
|
132
|
+
- lib/rdkafka/admin.rb
|
133
|
+
- lib/rdkafka/admin/create_topic_handle.rb
|
134
|
+
- lib/rdkafka/admin/create_topic_report.rb
|
135
|
+
- lib/rdkafka/admin/delete_topic_handle.rb
|
136
|
+
- lib/rdkafka/admin/delete_topic_report.rb
|
131
137
|
- lib/rdkafka/bindings.rb
|
138
|
+
- lib/rdkafka/callbacks.rb
|
132
139
|
- lib/rdkafka/config.rb
|
133
140
|
- lib/rdkafka/consumer.rb
|
134
141
|
- lib/rdkafka/consumer/headers.rb
|
@@ -136,18 +143,27 @@ files:
|
|
136
143
|
- lib/rdkafka/consumer/partition.rb
|
137
144
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
138
145
|
- lib/rdkafka/error.rb
|
146
|
+
- lib/rdkafka/metadata.rb
|
139
147
|
- lib/rdkafka/producer.rb
|
140
148
|
- lib/rdkafka/producer/delivery_handle.rb
|
141
149
|
- lib/rdkafka/producer/delivery_report.rb
|
142
150
|
- lib/rdkafka/version.rb
|
143
151
|
- rdkafka.gemspec
|
152
|
+
- spec/rdkafka/abstract_handle_spec.rb
|
153
|
+
- spec/rdkafka/admin/create_topic_handle_spec.rb
|
154
|
+
- spec/rdkafka/admin/create_topic_report_spec.rb
|
155
|
+
- spec/rdkafka/admin/delete_topic_handle_spec.rb
|
156
|
+
- spec/rdkafka/admin/delete_topic_report_spec.rb
|
157
|
+
- spec/rdkafka/admin_spec.rb
|
144
158
|
- spec/rdkafka/bindings_spec.rb
|
159
|
+
- spec/rdkafka/callbacks_spec.rb
|
145
160
|
- spec/rdkafka/config_spec.rb
|
146
161
|
- spec/rdkafka/consumer/message_spec.rb
|
147
162
|
- spec/rdkafka/consumer/partition_spec.rb
|
148
163
|
- spec/rdkafka/consumer/topic_partition_list_spec.rb
|
149
164
|
- spec/rdkafka/consumer_spec.rb
|
150
165
|
- spec/rdkafka/error_spec.rb
|
166
|
+
- spec/rdkafka/metadata_spec.rb
|
151
167
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
152
168
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
153
169
|
- spec/rdkafka/producer_spec.rb
|
@@ -156,7 +172,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
|
|
156
172
|
licenses:
|
157
173
|
- MIT
|
158
174
|
metadata: {}
|
159
|
-
post_install_message:
|
175
|
+
post_install_message:
|
160
176
|
rdoc_options: []
|
161
177
|
require_paths:
|
162
178
|
- lib
|
@@ -164,27 +180,35 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
164
180
|
requirements:
|
165
181
|
- - ">="
|
166
182
|
- !ruby/object:Gem::Version
|
167
|
-
version: '2.
|
183
|
+
version: '2.4'
|
168
184
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
169
185
|
requirements:
|
170
186
|
- - ">="
|
171
187
|
- !ruby/object:Gem::Version
|
172
188
|
version: '0'
|
173
189
|
requirements: []
|
174
|
-
|
175
|
-
|
176
|
-
signing_key:
|
190
|
+
rubygems_version: 3.2.3
|
191
|
+
signing_key:
|
177
192
|
specification_version: 4
|
178
|
-
summary:
|
179
|
-
|
193
|
+
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
194
|
+
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|
195
|
+
and Ruby 2.4+.
|
180
196
|
test_files:
|
197
|
+
- spec/rdkafka/abstract_handle_spec.rb
|
198
|
+
- spec/rdkafka/admin/create_topic_handle_spec.rb
|
199
|
+
- spec/rdkafka/admin/create_topic_report_spec.rb
|
200
|
+
- spec/rdkafka/admin/delete_topic_handle_spec.rb
|
201
|
+
- spec/rdkafka/admin/delete_topic_report_spec.rb
|
202
|
+
- spec/rdkafka/admin_spec.rb
|
181
203
|
- spec/rdkafka/bindings_spec.rb
|
204
|
+
- spec/rdkafka/callbacks_spec.rb
|
182
205
|
- spec/rdkafka/config_spec.rb
|
183
206
|
- spec/rdkafka/consumer/message_spec.rb
|
184
207
|
- spec/rdkafka/consumer/partition_spec.rb
|
185
208
|
- spec/rdkafka/consumer/topic_partition_list_spec.rb
|
186
209
|
- spec/rdkafka/consumer_spec.rb
|
187
210
|
- spec/rdkafka/error_spec.rb
|
211
|
+
- spec/rdkafka/metadata_spec.rb
|
188
212
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
189
213
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
190
214
|
- spec/rdkafka/producer_spec.rb
|
data/.travis.yml
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
sudo: false
|
4
|
-
|
5
|
-
services:
|
6
|
-
- docker
|
7
|
-
|
8
|
-
env:
|
9
|
-
global:
|
10
|
-
- CC_TEST_REPORTER_ID=9f7f740ac1b6e264e1189fa07a6687a87bcdb9f3c0f4199d4344ab3b538e187e
|
11
|
-
- KAFKA_HEAP_OPTS="-Xmx512m -Xms512m"
|
12
|
-
|
13
|
-
rvm:
|
14
|
-
- 2.3
|
15
|
-
- 2.4
|
16
|
-
- 2.5
|
17
|
-
- 2.6
|
18
|
-
|
19
|
-
before_install:
|
20
|
-
- gem update --system
|
21
|
-
|
22
|
-
before_script:
|
23
|
-
- docker-compose up -d
|
24
|
-
- cd ext && bundle exec rake && cd ..
|
25
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
26
|
-
- chmod +x ./cc-test-reporter
|
27
|
-
- ./cc-test-reporter before-build
|
28
|
-
|
29
|
-
script:
|
30
|
-
- bundle exec rspec
|
31
|
-
|
32
|
-
after_script:
|
33
|
-
- docker-compose stop
|
34
|
-
- ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
|