rdkafka 0.9.0 → 0.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/.semaphore/semaphore.yml +1 -1
- data/CHANGELOG.md +13 -1
- data/Guardfile +19 -0
- data/README.md +3 -1
- data/bin/console +11 -0
- data/docker-compose.yml +3 -3
- data/ext/README.md +1 -1
- data/ext/Rakefile +3 -19
- data/lib/rdkafka/bindings.rb +11 -0
- data/lib/rdkafka/config.rb +25 -0
- data/lib/rdkafka/consumer.rb +3 -3
- data/lib/rdkafka/producer/delivery_report.rb +1 -1
- data/lib/rdkafka/producer.rb +8 -2
- data/lib/rdkafka/version.rb +3 -3
- data/rdkafka.gemspec +9 -7
- data/spec/rdkafka/abstract_handle_spec.rb +0 -1
- data/spec/rdkafka/admin_spec.rb +1 -1
- data/spec/rdkafka/bindings_spec.rb +24 -0
- data/spec/rdkafka/config_spec.rb +33 -2
- data/spec/rdkafka/consumer_spec.rb +68 -44
- data/spec/rdkafka/metadata_spec.rb +2 -2
- data/spec/rdkafka/producer_spec.rb +4 -3
- data/spec/spec_helper.rb +50 -9
- metadata +57 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9b7042f241be5aad91c403f61aabb8e8ba87afa46674db8ba58c487fe01f88c9
|
4
|
+
data.tar.gz: 5cb00dc3dc8b4069a8e62cda9271e290d1371434332767270426488dedcff641
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b287a0d81aca7909702969778fb64585b6808860971d50e1c5540eb0c43cf73617f9e701605b816e8f52a81e39d9909512e1d113f6a550555e6ca503dc5ff58
|
7
|
+
data.tar.gz: 7924ab06f52646168b52d0478b605484e4aa1a3674ace4f97af5b56588aea6dec8e614d3258b0670f92f18a905509ae6982d2c8999fa719f8ea7451e515ecb3d
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--format documentation
|
data/.semaphore/semaphore.yml
CHANGED
@@ -13,7 +13,7 @@ blocks:
|
|
13
13
|
- name: bundle exec rspec
|
14
14
|
matrix:
|
15
15
|
- env_var: RUBY_VERSION
|
16
|
-
values: [ "2.
|
16
|
+
values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
|
17
17
|
commands:
|
18
18
|
- sem-version ruby $RUBY_VERSION
|
19
19
|
- checkout
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
# 0.11.1
|
2
|
+
* Use mini_portile2 2.6, otherwise you can't run nokogiri and rdkafka at the same time.
|
3
|
+
|
4
|
+
# 0.11.0
|
5
|
+
* Upgrade librdkafka to 1.8.2
|
6
|
+
* Bump supported minimum Ruby version to 2.6
|
7
|
+
* Better homebrew path detection
|
8
|
+
|
9
|
+
# 0.10.0
|
10
|
+
* Upgrade librdkafka to 1.5.0
|
11
|
+
* Add error callback config
|
12
|
+
|
1
13
|
# 0.9.0
|
2
14
|
* Fixes for Ruby 3.0
|
3
15
|
* Allow any callable object for callbacks (gremerritt)
|
@@ -41,7 +53,7 @@
|
|
41
53
|
* Use default Homebrew openssl location if present
|
42
54
|
* Consumer lag handles empty topics
|
43
55
|
* End iteration in consumer when it is closed
|
44
|
-
* Add
|
56
|
+
* Add support for storing message offsets
|
45
57
|
* Add missing runtime dependency to rake
|
46
58
|
|
47
59
|
# 0.4.1
|
data/Guardfile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
logger level: :error
|
4
|
+
|
5
|
+
guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
|
6
|
+
require "guard/rspec/dsl"
|
7
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
8
|
+
|
9
|
+
# Ruby files
|
10
|
+
ruby = dsl.ruby
|
11
|
+
dsl.watch_spec_files_for(ruby.lib_files)
|
12
|
+
watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
|
13
|
+
|
14
|
+
# RSpec files
|
15
|
+
rspec = dsl.rspec
|
16
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
17
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
18
|
+
watch(rspec.spec_files)
|
19
|
+
end
|
data/README.md
CHANGED
@@ -7,7 +7,9 @@
|
|
7
7
|
The `rdkafka` gem is a modern Kafka client library for Ruby based on
|
8
8
|
[librdkafka](https://github.com/edenhill/librdkafka/).
|
9
9
|
It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
|
10
|
-
gem and targets Kafka 1.0+ and Ruby
|
10
|
+
gem and targets Kafka 1.0+ and Ruby versions that are under security or
|
11
|
+
active maintenance. We remove Ruby version from our CI builds if they
|
12
|
+
become EOL.
|
11
13
|
|
12
14
|
`rdkafka` was written because we needed a reliable Ruby client for
|
13
15
|
Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
|
data/bin/console
ADDED
data/docker-compose.yml
CHANGED
@@ -4,13 +4,13 @@ version: '2'
|
|
4
4
|
|
5
5
|
services:
|
6
6
|
zookeeper:
|
7
|
-
image: confluentinc/cp-zookeeper:
|
7
|
+
image: confluentinc/cp-zookeeper:5.2.6
|
8
8
|
environment:
|
9
9
|
ZOOKEEPER_CLIENT_PORT: 2181
|
10
10
|
ZOOKEEPER_TICK_TIME: 2000
|
11
11
|
|
12
12
|
kafka:
|
13
|
-
image: confluentinc/cp-kafka:
|
13
|
+
image: confluentinc/cp-kafka:5.2.5-10
|
14
14
|
depends_on:
|
15
15
|
- zookeeper
|
16
16
|
ports:
|
@@ -18,7 +18,7 @@ services:
|
|
18
18
|
environment:
|
19
19
|
KAFKA_BROKER_ID: 1
|
20
20
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
21
|
-
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://
|
21
|
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
|
22
22
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
23
23
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
24
24
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
data/ext/README.md
CHANGED
data/ext/Rakefile
CHANGED
@@ -4,30 +4,14 @@ require "fileutils"
|
|
4
4
|
require "open-uri"
|
5
5
|
|
6
6
|
task :default => :clean do
|
7
|
-
# MiniPortile#download_file_http is a monkey patch that removes the download
|
8
|
-
# progress indicator. This indicator relies on the 'Content Length' response
|
9
|
-
# headers, which is not set by GitHub
|
10
|
-
class MiniPortile
|
11
|
-
def download_file_http(url, full_path, _count)
|
12
|
-
filename = File.basename(full_path)
|
13
|
-
with_tempfile(filename, full_path) do |temp_file|
|
14
|
-
params = { 'Accept-Encoding' => 'identity' }
|
15
|
-
OpenURI.open_uri(url, 'rb', params) do |io|
|
16
|
-
temp_file.write(io.read)
|
17
|
-
end
|
18
|
-
output
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
7
|
# Download and compile librdkafka
|
24
8
|
recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
|
25
9
|
|
26
10
|
# Use default homebrew openssl if we're on mac and the directory exists
|
27
11
|
# and each of flags is not empty
|
28
|
-
if recipe.host&.include?("darwin") && Dir.exist?("
|
29
|
-
ENV["CPPFLAGS"] = "-I/
|
30
|
-
ENV["LDFLAGS"] = "-L/
|
12
|
+
if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
|
13
|
+
ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
|
14
|
+
ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
|
31
15
|
end
|
32
16
|
|
33
17
|
recipe.files << {
|
data/lib/rdkafka/bindings.rb
CHANGED
@@ -108,6 +108,8 @@ module Rdkafka
|
|
108
108
|
attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
|
109
109
|
callback :stats_cb, [:pointer, :string, :int, :pointer], :int
|
110
110
|
attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
|
111
|
+
callback :error_cb, [:pointer, :int, :string, :pointer], :void
|
112
|
+
attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
|
111
113
|
|
112
114
|
# Log queue
|
113
115
|
attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
|
@@ -146,6 +148,15 @@ module Rdkafka
|
|
146
148
|
0
|
147
149
|
end
|
148
150
|
|
151
|
+
ErrorCallback = FFI::Function.new(
|
152
|
+
:void, [:pointer, :int, :string, :pointer]
|
153
|
+
) do |_client_prr, err_code, reason, _opaque|
|
154
|
+
if Rdkafka::Config.error_callback
|
155
|
+
error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
156
|
+
Rdkafka::Config.error_callback.call(error)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
149
160
|
# Handle
|
150
161
|
|
151
162
|
enum :kafka_type, [
|
data/lib/rdkafka/config.rb
CHANGED
@@ -10,6 +10,8 @@ module Rdkafka
|
|
10
10
|
# @private
|
11
11
|
@@statistics_callback = nil
|
12
12
|
# @private
|
13
|
+
@@error_callback = nil
|
14
|
+
# @private
|
13
15
|
@@opaques = {}
|
14
16
|
# @private
|
15
17
|
@@log_queue = Queue.new
|
@@ -28,6 +30,7 @@ module Rdkafka
|
|
28
30
|
@@logger
|
29
31
|
end
|
30
32
|
|
33
|
+
|
31
34
|
# Returns a queue whose contents will be passed to the configured logger. Each entry
|
32
35
|
# should follow the format [Logger::Severity, String]. The benefit over calling the
|
33
36
|
# logger directly is that this is safe to use from trap contexts.
|
@@ -66,6 +69,25 @@ module Rdkafka
|
|
66
69
|
@@statistics_callback
|
67
70
|
end
|
68
71
|
|
72
|
+
# Set a callback that will be called every time the underlying client emits an error.
|
73
|
+
# If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
|
74
|
+
# The callback is called with an instance of RdKafka::Error.
|
75
|
+
#
|
76
|
+
# @param callback [Proc, #call] The callback
|
77
|
+
#
|
78
|
+
# @return [nil]
|
79
|
+
def self.error_callback=(callback)
|
80
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
|
81
|
+
@@error_callback = callback
|
82
|
+
end
|
83
|
+
|
84
|
+
# Returns the current error callback, by default this is nil.
|
85
|
+
#
|
86
|
+
# @return [Proc, nil]
|
87
|
+
def self.error_callback
|
88
|
+
@@error_callback
|
89
|
+
end
|
90
|
+
|
69
91
|
# @private
|
70
92
|
def self.opaques
|
71
93
|
@@opaques
|
@@ -221,6 +243,9 @@ module Rdkafka
|
|
221
243
|
|
222
244
|
# Set stats callback
|
223
245
|
Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
|
246
|
+
|
247
|
+
# Set error callback
|
248
|
+
Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
|
224
249
|
end
|
225
250
|
end
|
226
251
|
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -498,11 +498,11 @@ module Rdkafka
|
|
498
498
|
# Exception behavior is more complicated than with `each`, in that if
|
499
499
|
# :yield_on_error is true, and an exception is raised during the
|
500
500
|
# poll, and messages have already been received, they will be yielded to
|
501
|
-
# the caller before the exception is allowed to
|
501
|
+
# the caller before the exception is allowed to propagate.
|
502
502
|
#
|
503
503
|
# If you are setting either auto.commit or auto.offset.store to false in
|
504
504
|
# the consumer configuration, then you should let yield_on_error keep its
|
505
|
-
# default value of false because you are
|
505
|
+
# default value of false because you are guaranteed to see these messages
|
506
506
|
# again. However, if both auto.commit and auto.offset.store are set to
|
507
507
|
# true, you should set yield_on_error to true so you can process messages
|
508
508
|
# that you may or may not see again.
|
@@ -518,7 +518,7 @@ module Rdkafka
|
|
518
518
|
# @yield [messages, pending_exception]
|
519
519
|
# @yieldparam messages [Array] An array of received Message
|
520
520
|
# @yieldparam pending_exception [Exception] normally nil, or an exception
|
521
|
-
# which will be
|
521
|
+
# which will be propagated after processing of the partial batch is complete.
|
522
522
|
#
|
523
523
|
# @return [nil]
|
524
524
|
def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require "securerandom"
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
|
3
5
|
class Producer
|
@@ -9,11 +11,12 @@ module Rdkafka
|
|
9
11
|
|
10
12
|
# @private
|
11
13
|
def initialize(native_kafka)
|
14
|
+
@id = SecureRandom.uuid
|
12
15
|
@closing = false
|
13
16
|
@native_kafka = native_kafka
|
14
17
|
|
15
18
|
# Makes sure, that the producer gets closed before it gets GCed by Ruby
|
16
|
-
ObjectSpace.define_finalizer(
|
19
|
+
ObjectSpace.define_finalizer(@id, proc { close })
|
17
20
|
|
18
21
|
# Start thread to poll client for delivery callbacks
|
19
22
|
@polling_thread = Thread.new do
|
@@ -41,6 +44,8 @@ module Rdkafka
|
|
41
44
|
|
42
45
|
# Close this producer and wait for the internal poll queue to empty.
|
43
46
|
def close
|
47
|
+
ObjectSpace.undefine_finalizer(@id)
|
48
|
+
|
44
49
|
return unless @native_kafka
|
45
50
|
|
46
51
|
# Indicate to polling thread that we're closing
|
@@ -70,8 +75,9 @@ module Rdkafka
|
|
70
75
|
#
|
71
76
|
# @param topic [String] The topic to produce to
|
72
77
|
# @param payload [String,nil] The message's payload
|
73
|
-
# @param key [String] The message's key
|
78
|
+
# @param key [String, nil] The message's key
|
74
79
|
# @param partition [Integer,nil] Optional partition to produce to
|
80
|
+
# @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
|
75
81
|
# @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
|
76
82
|
# @param headers [Hash<String,String>] Optional message headers
|
77
83
|
#
|
data/lib/rdkafka/version.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Rdkafka
|
2
|
-
VERSION = "0.
|
3
|
-
LIBRDKAFKA_VERSION = "1.
|
4
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
2
|
+
VERSION = "0.11.1"
|
3
|
+
LIBRDKAFKA_VERSION = "1.8.2"
|
4
|
+
LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
|
5
5
|
end
|
data/rdkafka.gemspec
CHANGED
@@ -14,15 +14,17 @@ Gem::Specification.new do |gem|
|
|
14
14
|
gem.name = 'rdkafka'
|
15
15
|
gem.require_paths = ['lib']
|
16
16
|
gem.version = Rdkafka::VERSION
|
17
|
-
gem.required_ruby_version = '>= 2.
|
17
|
+
gem.required_ruby_version = '>= 2.6'
|
18
18
|
gem.extensions = %w(ext/Rakefile)
|
19
19
|
|
20
|
-
gem.add_dependency 'ffi', '~> 1.
|
21
|
-
gem.add_dependency 'mini_portile2', '~> 2.
|
22
|
-
gem.add_dependency 'rake', '
|
20
|
+
gem.add_dependency 'ffi', '~> 1.15'
|
21
|
+
gem.add_dependency 'mini_portile2', '~> 2.6'
|
22
|
+
gem.add_dependency 'rake', '> 12'
|
23
23
|
|
24
|
-
gem.add_development_dependency 'pry'
|
24
|
+
gem.add_development_dependency 'pry'
|
25
25
|
gem.add_development_dependency 'rspec', '~> 3.5'
|
26
|
-
gem.add_development_dependency 'rake'
|
27
|
-
gem.add_development_dependency 'simplecov'
|
26
|
+
gem.add_development_dependency 'rake'
|
27
|
+
gem.add_development_dependency 'simplecov'
|
28
|
+
gem.add_development_dependency 'guard'
|
29
|
+
gem.add_development_dependency 'guard-rspec'
|
28
30
|
end
|
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -100,4 +100,28 @@ describe Rdkafka::Bindings do
|
|
100
100
|
end
|
101
101
|
end
|
102
102
|
end
|
103
|
+
|
104
|
+
describe "error callback" do
|
105
|
+
context "without an error callback" do
|
106
|
+
it "should do nothing" do
|
107
|
+
expect {
|
108
|
+
Rdkafka::Bindings::ErrorCallback.call(nil, 1, "error", nil)
|
109
|
+
}.not_to raise_error
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
context "with an error callback" do
|
114
|
+
before do
|
115
|
+
Rdkafka::Config.error_callback = lambda do |error|
|
116
|
+
$received_error = error
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
it "should call the error callback with an Rdkafka::Error" do
|
121
|
+
Rdkafka::Bindings::ErrorCallback.call(nil, 8, "Broker not available", nil)
|
122
|
+
expect($received_error.code).to eq(:broker_not_available)
|
123
|
+
expect($received_error.broker_message).to eq("Broker not available")
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
103
127
|
end
|
data/spec/rdkafka/config_spec.rb
CHANGED
@@ -64,6 +64,37 @@ describe Rdkafka::Config do
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
+
context "error callback" do
|
68
|
+
context "with a proc/lambda" do
|
69
|
+
it "should set the callback" do
|
70
|
+
expect {
|
71
|
+
Rdkafka::Config.error_callback = lambda do |error|
|
72
|
+
puts error
|
73
|
+
end
|
74
|
+
}.not_to raise_error
|
75
|
+
expect(Rdkafka::Config.error_callback).to respond_to :call
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
context "with a callable object" do
|
80
|
+
it "should set the callback" do
|
81
|
+
callback = Class.new do
|
82
|
+
def call(stats); end
|
83
|
+
end
|
84
|
+
expect {
|
85
|
+
Rdkafka::Config.error_callback = callback.new
|
86
|
+
}.not_to raise_error
|
87
|
+
expect(Rdkafka::Config.error_callback).to respond_to :call
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should not accept a callback that's not callable" do
|
92
|
+
expect {
|
93
|
+
Rdkafka::Config.error_callback = 'a string'
|
94
|
+
}.to raise_error(TypeError)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
67
98
|
context "configuration" do
|
68
99
|
it "should store configuration" do
|
69
100
|
config = Rdkafka::Config.new
|
@@ -77,7 +108,7 @@ describe Rdkafka::Config do
|
|
77
108
|
end
|
78
109
|
|
79
110
|
it "should create a consumer with valid config" do
|
80
|
-
consumer =
|
111
|
+
consumer = rdkafka_consumer_config.consumer
|
81
112
|
expect(consumer).to be_a Rdkafka::Consumer
|
82
113
|
consumer.close
|
83
114
|
end
|
@@ -105,7 +136,7 @@ describe Rdkafka::Config do
|
|
105
136
|
end
|
106
137
|
|
107
138
|
it "should create a producer with valid config" do
|
108
|
-
producer =
|
139
|
+
producer = rdkafka_consumer_config.producer
|
109
140
|
expect(producer).to be_a Rdkafka::Producer
|
110
141
|
producer.close
|
111
142
|
end
|
@@ -3,9 +3,8 @@ require "ostruct"
|
|
3
3
|
require 'securerandom'
|
4
4
|
|
5
5
|
describe Rdkafka::Consumer do
|
6
|
-
let(:
|
7
|
-
let(:
|
8
|
-
let(:producer) { config.producer }
|
6
|
+
let(:consumer) { rdkafka_consumer_config.consumer }
|
7
|
+
let(:producer) { rdkafka_producer_config.producer }
|
9
8
|
|
10
9
|
after { consumer.close }
|
11
10
|
after { producer.close }
|
@@ -328,7 +327,7 @@ describe Rdkafka::Consumer do
|
|
328
327
|
before :all do
|
329
328
|
# Make sure there are some messages.
|
330
329
|
handles = []
|
331
|
-
producer =
|
330
|
+
producer = rdkafka_producer_config.producer
|
332
331
|
10.times do
|
333
332
|
(0..2).each do |i|
|
334
333
|
handles << producer.produce(
|
@@ -404,7 +403,7 @@ describe Rdkafka::Consumer do
|
|
404
403
|
config = {}
|
405
404
|
config[:'enable.auto.offset.store'] = false
|
406
405
|
config[:'enable.auto.commit'] = false
|
407
|
-
@new_consumer =
|
406
|
+
@new_consumer = rdkafka_consumer_config(config).consumer
|
408
407
|
@new_consumer.subscribe("consume_test_topic")
|
409
408
|
wait_for_assignment(@new_consumer)
|
410
409
|
end
|
@@ -459,13 +458,13 @@ describe Rdkafka::Consumer do
|
|
459
458
|
end
|
460
459
|
|
461
460
|
describe "#lag" do
|
462
|
-
let(:
|
461
|
+
let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
|
463
462
|
|
464
463
|
it "should calculate the consumer lag" do
|
465
464
|
# Make sure there's a message in every partition and
|
466
465
|
# wait for the message to make sure everything is committed.
|
467
466
|
(0..2).each do |i|
|
468
|
-
|
467
|
+
producer.produce(
|
469
468
|
topic: "consume_test_topic",
|
470
469
|
key: "key lag #{i}",
|
471
470
|
partition: i
|
@@ -508,7 +507,7 @@ describe Rdkafka::Consumer do
|
|
508
507
|
|
509
508
|
# Produce message on every topic again
|
510
509
|
(0..2).each do |i|
|
511
|
-
|
510
|
+
producer.produce(
|
512
511
|
topic: "consume_test_topic",
|
513
512
|
key: "key lag #{i}",
|
514
513
|
partition: i
|
@@ -724,6 +723,8 @@ describe Rdkafka::Consumer do
|
|
724
723
|
#
|
725
724
|
# This is, in effect, an integration test and the subsequent specs are
|
726
725
|
# unit tests.
|
726
|
+
create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
|
727
|
+
create_topic_handle.wait(max_wait_timeout: 15.0)
|
727
728
|
consumer.subscribe(topic_name)
|
728
729
|
produce_n 42
|
729
730
|
all_yields = []
|
@@ -777,6 +778,8 @@ describe Rdkafka::Consumer do
|
|
777
778
|
end
|
778
779
|
|
779
780
|
it "should yield [] if nothing is received before the timeout" do
|
781
|
+
create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
|
782
|
+
create_topic_handle.wait(max_wait_timeout: 15.0)
|
780
783
|
consumer.subscribe(topic_name)
|
781
784
|
consumer.each_batch do |batch|
|
782
785
|
expect(batch).to eq([])
|
@@ -820,8 +823,12 @@ describe Rdkafka::Consumer do
|
|
820
823
|
|
821
824
|
context "error raised from poll and yield_on_error is true" do
|
822
825
|
it "should yield buffered exceptions on rebalance, then break" do
|
823
|
-
config =
|
824
|
-
|
826
|
+
config = rdkafka_consumer_config(
|
827
|
+
{
|
828
|
+
:"enable.auto.commit" => false,
|
829
|
+
:"enable.auto.offset.store" => false
|
830
|
+
}
|
831
|
+
)
|
825
832
|
consumer = config.consumer
|
826
833
|
consumer.subscribe(topic_name)
|
827
834
|
loop_count = 0
|
@@ -860,8 +867,12 @@ describe Rdkafka::Consumer do
|
|
860
867
|
|
861
868
|
context "error raised from poll and yield_on_error is false" do
|
862
869
|
it "should yield buffered exceptions on rebalance, then break" do
|
863
|
-
config =
|
864
|
-
|
870
|
+
config = rdkafka_consumer_config(
|
871
|
+
{
|
872
|
+
:"enable.auto.commit" => false,
|
873
|
+
:"enable.auto.offset.store" => false
|
874
|
+
}
|
875
|
+
)
|
865
876
|
consumer = config.consumer
|
866
877
|
consumer.subscribe(topic_name)
|
867
878
|
loop_count = 0
|
@@ -898,51 +909,64 @@ describe Rdkafka::Consumer do
|
|
898
909
|
end
|
899
910
|
|
900
911
|
describe "a rebalance listener" do
|
901
|
-
|
902
|
-
|
903
|
-
|
904
|
-
|
905
|
-
|
912
|
+
let(:consumer) do
|
913
|
+
config = rdkafka_consumer_config
|
914
|
+
config.consumer_rebalance_listener = listener
|
915
|
+
config.consumer
|
916
|
+
end
|
906
917
|
|
907
|
-
|
908
|
-
|
909
|
-
|
918
|
+
context "with a working listener" do
|
919
|
+
let(:listener) do
|
920
|
+
Struct.new(:queue) do
|
921
|
+
def on_partitions_assigned(consumer, list)
|
922
|
+
collect(:assign, list)
|
923
|
+
end
|
910
924
|
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
915
|
-
|
925
|
+
def on_partitions_revoked(consumer, list)
|
926
|
+
collect(:revoke, list)
|
927
|
+
end
|
928
|
+
|
929
|
+
def collect(name, list)
|
930
|
+
partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
|
931
|
+
queue << ([name] + partitions)
|
932
|
+
end
|
933
|
+
end.new([])
|
934
|
+
end
|
916
935
|
|
917
|
-
|
936
|
+
it "should get notifications" do
|
937
|
+
notify_listener(listener)
|
918
938
|
|
919
|
-
|
920
|
-
|
921
|
-
|
922
|
-
|
939
|
+
expect(listener.queue).to eq([
|
940
|
+
[:assign, "consume_test_topic", 0, 1, 2],
|
941
|
+
[:revoke, "consume_test_topic", 0, 1, 2]
|
942
|
+
])
|
943
|
+
end
|
923
944
|
end
|
924
945
|
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
946
|
+
context "with a broken listener" do
|
947
|
+
let(:listener) do
|
948
|
+
Struct.new(:queue) do
|
949
|
+
def on_partitions_assigned(consumer, list)
|
950
|
+
queue << :assigned
|
951
|
+
raise 'boom'
|
952
|
+
end
|
931
953
|
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
954
|
+
def on_partitions_revoked(consumer, list)
|
955
|
+
queue << :revoked
|
956
|
+
raise 'boom'
|
957
|
+
end
|
958
|
+
end.new([])
|
959
|
+
end
|
937
960
|
|
938
|
-
|
961
|
+
it 'should handle callback exceptions' do
|
962
|
+
notify_listener(listener)
|
939
963
|
|
940
|
-
|
964
|
+
expect(listener.queue).to eq([:assigned, :revoked])
|
965
|
+
end
|
941
966
|
end
|
942
967
|
|
943
968
|
def notify_listener(listener)
|
944
969
|
# 1. subscribe and poll
|
945
|
-
config.consumer_rebalance_listener = listener
|
946
970
|
consumer.subscribe("consume_test_topic")
|
947
971
|
wait_for_assignment(consumer)
|
948
972
|
consumer.poll(100)
|
@@ -2,7 +2,7 @@ require "spec_helper"
|
|
2
2
|
require "securerandom"
|
3
3
|
|
4
4
|
describe Rdkafka::Metadata do
|
5
|
-
let(:config) {
|
5
|
+
let(:config) { rdkafka_consumer_config }
|
6
6
|
let(:native_config) { config.send(:native_config) }
|
7
7
|
let(:native_kafka) { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
|
8
8
|
|
@@ -18,7 +18,7 @@ describe Rdkafka::Metadata do
|
|
18
18
|
it "raises an appropriate exception" do
|
19
19
|
expect {
|
20
20
|
described_class.new(native_kafka, topic_name)
|
21
|
-
}.to raise_exception(Rdkafka::RdkafkaError, "Broker:
|
21
|
+
}.to raise_exception(Rdkafka::RdkafkaError, "Broker: Unknown topic or partition (unknown_topic_or_part)")
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
@@ -1,8 +1,9 @@
|
|
1
1
|
require "spec_helper"
|
2
|
+
require "zlib"
|
2
3
|
|
3
4
|
describe Rdkafka::Producer do
|
4
|
-
let(:producer) {
|
5
|
-
let(:consumer) {
|
5
|
+
let(:producer) { rdkafka_producer_config.producer }
|
6
|
+
let(:consumer) { rdkafka_consumer_config.consumer }
|
6
7
|
|
7
8
|
after do
|
8
9
|
# Registry should always end up being empty
|
@@ -388,7 +389,7 @@ describe Rdkafka::Producer do
|
|
388
389
|
reader.close
|
389
390
|
|
390
391
|
# Avoids sharing the socket between processes.
|
391
|
-
producer =
|
392
|
+
producer = rdkafka_producer_config.producer
|
392
393
|
|
393
394
|
handle = producer.produce(
|
394
395
|
topic: "produce_test_topic",
|
data/spec/spec_helper.rb
CHANGED
@@ -8,27 +8,57 @@ end
|
|
8
8
|
require "pry"
|
9
9
|
require "rspec"
|
10
10
|
require "rdkafka"
|
11
|
+
require "timeout"
|
11
12
|
|
12
|
-
def
|
13
|
-
|
13
|
+
def rdkafka_base_config
|
14
|
+
{
|
14
15
|
:"api.version.request" => false,
|
15
16
|
:"broker.version.fallback" => "1.0",
|
16
17
|
:"bootstrap.servers" => "localhost:9092",
|
17
|
-
:"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
|
18
|
-
:"auto.offset.reset" => "earliest",
|
19
|
-
:"enable.partition.eof" => false
|
20
18
|
}
|
19
|
+
end
|
20
|
+
|
21
|
+
def rdkafka_config(config_overrides={})
|
22
|
+
# Generate the base config
|
23
|
+
config = rdkafka_base_config
|
24
|
+
# Merge overrides
|
25
|
+
config.merge!(config_overrides)
|
26
|
+
# Return it
|
27
|
+
Rdkafka::Config.new(config)
|
28
|
+
end
|
29
|
+
|
30
|
+
def rdkafka_consumer_config(config_overrides={})
|
31
|
+
# Generate the base config
|
32
|
+
config = rdkafka_base_config
|
33
|
+
# Add consumer specific fields to it
|
34
|
+
config[:"auto.offset.reset"] = "earliest"
|
35
|
+
config[:"enable.partition.eof"] = false
|
36
|
+
config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
|
37
|
+
# Enable debug mode if required
|
38
|
+
if ENV["DEBUG_CONSUMER"]
|
39
|
+
config[:debug] = "cgrp,topic,fetch"
|
40
|
+
end
|
41
|
+
# Merge overrides
|
42
|
+
config.merge!(config_overrides)
|
43
|
+
# Return it
|
44
|
+
Rdkafka::Config.new(config)
|
45
|
+
end
|
46
|
+
|
47
|
+
def rdkafka_producer_config(config_overrides={})
|
48
|
+
# Generate the base config
|
49
|
+
config = rdkafka_base_config
|
50
|
+
# Enable debug mode if required
|
21
51
|
if ENV["DEBUG_PRODUCER"]
|
22
52
|
config[:debug] = "broker,topic,msg"
|
23
|
-
elsif ENV["DEBUG_CONSUMER"]
|
24
|
-
config[:debug] = "cgrp,topic,fetch"
|
25
53
|
end
|
54
|
+
# Merge overrides
|
26
55
|
config.merge!(config_overrides)
|
56
|
+
# Return it
|
27
57
|
Rdkafka::Config.new(config)
|
28
58
|
end
|
29
59
|
|
30
60
|
def new_native_client
|
31
|
-
config =
|
61
|
+
config = rdkafka_consumer_config
|
32
62
|
config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
|
33
63
|
end
|
34
64
|
|
@@ -42,7 +72,7 @@ end
|
|
42
72
|
|
43
73
|
def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
|
44
74
|
new_consumer = !!consumer
|
45
|
-
consumer ||=
|
75
|
+
consumer ||= rdkafka_consumer_config.consumer
|
46
76
|
consumer.subscribe(topic)
|
47
77
|
timeout = Time.now.to_i + timeout_in_seconds
|
48
78
|
loop do
|
@@ -75,6 +105,9 @@ def wait_for_unassignment(consumer)
|
|
75
105
|
end
|
76
106
|
|
77
107
|
RSpec.configure do |config|
|
108
|
+
config.filter_run focus: true
|
109
|
+
config.run_all_when_everything_filtered = true
|
110
|
+
|
78
111
|
config.before(:suite) do
|
79
112
|
admin = rdkafka_config.admin
|
80
113
|
{
|
@@ -95,4 +128,12 @@ RSpec.configure do |config|
|
|
95
128
|
end
|
96
129
|
admin.close
|
97
130
|
end
|
131
|
+
|
132
|
+
config.around(:each) do |example|
|
133
|
+
# Timeout specs after a minute. If they take longer
|
134
|
+
# they are probably stuck
|
135
|
+
Timeout::timeout(60) do
|
136
|
+
example.run
|
137
|
+
end
|
138
|
+
end
|
98
139
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -16,56 +16,56 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.15'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.15'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: mini_portile2
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '2.
|
33
|
+
version: '2.6'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '2.
|
40
|
+
version: '2.6'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '12
|
47
|
+
version: '12'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '12
|
54
|
+
version: '12'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: pry
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0
|
61
|
+
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '0
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,46 +84,78 @@ dependencies:
|
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: simplecov
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - "
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: '0
|
103
|
+
version: '0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - "
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: guard
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: guard-rspec
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
109
137
|
- !ruby/object:Gem::Version
|
110
|
-
version: '0
|
138
|
+
version: '0'
|
111
139
|
description: Modern Kafka client library for Ruby based on librdkafka
|
112
140
|
email:
|
113
141
|
- thijs@appsignal.com
|
114
|
-
executables:
|
142
|
+
executables:
|
143
|
+
- console
|
115
144
|
extensions:
|
116
145
|
- ext/Rakefile
|
117
146
|
extra_rdoc_files: []
|
118
147
|
files:
|
119
148
|
- ".gitignore"
|
149
|
+
- ".rspec"
|
120
150
|
- ".semaphore/semaphore.yml"
|
121
151
|
- ".yardopts"
|
122
152
|
- CHANGELOG.md
|
123
153
|
- Gemfile
|
154
|
+
- Guardfile
|
124
155
|
- LICENSE
|
125
156
|
- README.md
|
126
157
|
- Rakefile
|
158
|
+
- bin/console
|
127
159
|
- docker-compose.yml
|
128
160
|
- ext/README.md
|
129
161
|
- ext/Rakefile
|
@@ -180,14 +212,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
180
212
|
requirements:
|
181
213
|
- - ">="
|
182
214
|
- !ruby/object:Gem::Version
|
183
|
-
version: '2.
|
215
|
+
version: '2.6'
|
184
216
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
185
217
|
requirements:
|
186
218
|
- - ">="
|
187
219
|
- !ruby/object:Gem::Version
|
188
220
|
version: '0'
|
189
221
|
requirements: []
|
190
|
-
rubygems_version: 3.
|
222
|
+
rubygems_version: 3.1.4
|
191
223
|
signing_key:
|
192
224
|
specification_version: 4
|
193
225
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|