rdkafka 0.10.0 → 0.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/.semaphore/semaphore.yml +1 -1
- data/CHANGELOG.md +9 -1
- data/Guardfile +19 -0
- data/README.md +3 -1
- data/bin/console +11 -0
- data/docker-compose.yml +3 -3
- data/ext/README.md +1 -1
- data/ext/Rakefile +3 -19
- data/lib/rdkafka/consumer.rb +3 -3
- data/lib/rdkafka/producer/delivery_report.rb +1 -1
- data/lib/rdkafka/producer.rb +2 -1
- data/lib/rdkafka/version.rb +3 -3
- data/rdkafka.gemspec +9 -7
- data/spec/rdkafka/abstract_handle_spec.rb +0 -1
- data/spec/rdkafka/admin_spec.rb +1 -1
- data/spec/rdkafka/config_spec.rb +2 -2
- data/spec/rdkafka/consumer_spec.rb +64 -44
- data/spec/rdkafka/metadata_spec.rb +1 -1
- data/spec/rdkafka/producer_spec.rb +4 -3
- data/spec/spec_helper.rb +50 -9
- metadata +56 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9b7042f241be5aad91c403f61aabb8e8ba87afa46674db8ba58c487fe01f88c9
|
4
|
+
data.tar.gz: 5cb00dc3dc8b4069a8e62cda9271e290d1371434332767270426488dedcff641
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b287a0d81aca7909702969778fb64585b6808860971d50e1c5540eb0c43cf73617f9e701605b816e8f52a81e39d9909512e1d113f6a550555e6ca503dc5ff58
|
7
|
+
data.tar.gz: 7924ab06f52646168b52d0478b605484e4aa1a3674ace4f97af5b56588aea6dec8e614d3258b0670f92f18a905509ae6982d2c8999fa719f8ea7451e515ecb3d
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--format documentation
|
data/.semaphore/semaphore.yml
CHANGED
@@ -13,7 +13,7 @@ blocks:
|
|
13
13
|
- name: bundle exec rspec
|
14
14
|
matrix:
|
15
15
|
- env_var: RUBY_VERSION
|
16
|
-
values: [ "2.
|
16
|
+
values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
|
17
17
|
commands:
|
18
18
|
- sem-version ruby $RUBY_VERSION
|
19
19
|
- checkout
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
# 0.11.1
|
2
|
+
* Use mini_portile2 2.6, otherwise you can't run nokogiri and rdkafka at the same time.
|
3
|
+
|
4
|
+
# 0.11.0
|
5
|
+
* Upgrade librdkafka to 1.8.2
|
6
|
+
* Bump supported minimum Ruby version to 2.6
|
7
|
+
* Better homebrew path detection
|
8
|
+
|
1
9
|
# 0.10.0
|
2
10
|
* Upgrade librdkafka to 1.5.0
|
3
11
|
* Add error callback config
|
@@ -45,7 +53,7 @@
|
|
45
53
|
* Use default Homebrew openssl location if present
|
46
54
|
* Consumer lag handles empty topics
|
47
55
|
* End iteration in consumer when it is closed
|
48
|
-
* Add
|
56
|
+
* Add support for storing message offsets
|
49
57
|
* Add missing runtime dependency to rake
|
50
58
|
|
51
59
|
# 0.4.1
|
data/Guardfile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
logger level: :error
|
4
|
+
|
5
|
+
guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
|
6
|
+
require "guard/rspec/dsl"
|
7
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
8
|
+
|
9
|
+
# Ruby files
|
10
|
+
ruby = dsl.ruby
|
11
|
+
dsl.watch_spec_files_for(ruby.lib_files)
|
12
|
+
watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
|
13
|
+
|
14
|
+
# RSpec files
|
15
|
+
rspec = dsl.rspec
|
16
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
17
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
18
|
+
watch(rspec.spec_files)
|
19
|
+
end
|
data/README.md
CHANGED
@@ -7,7 +7,9 @@
|
|
7
7
|
The `rdkafka` gem is a modern Kafka client library for Ruby based on
|
8
8
|
[librdkafka](https://github.com/edenhill/librdkafka/).
|
9
9
|
It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
|
10
|
-
gem and targets Kafka 1.0+ and Ruby
|
10
|
+
gem and targets Kafka 1.0+ and Ruby versions that are under security or
|
11
|
+
active maintenance. We remove Ruby version from our CI builds if they
|
12
|
+
become EOL.
|
11
13
|
|
12
14
|
`rdkafka` was written because we needed a reliable Ruby client for
|
13
15
|
Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
|
data/bin/console
ADDED
data/docker-compose.yml
CHANGED
@@ -4,13 +4,13 @@ version: '2'
|
|
4
4
|
|
5
5
|
services:
|
6
6
|
zookeeper:
|
7
|
-
image: confluentinc/cp-zookeeper:
|
7
|
+
image: confluentinc/cp-zookeeper:5.2.6
|
8
8
|
environment:
|
9
9
|
ZOOKEEPER_CLIENT_PORT: 2181
|
10
10
|
ZOOKEEPER_TICK_TIME: 2000
|
11
11
|
|
12
12
|
kafka:
|
13
|
-
image: confluentinc/cp-kafka:
|
13
|
+
image: confluentinc/cp-kafka:5.2.5-10
|
14
14
|
depends_on:
|
15
15
|
- zookeeper
|
16
16
|
ports:
|
@@ -18,7 +18,7 @@ services:
|
|
18
18
|
environment:
|
19
19
|
KAFKA_BROKER_ID: 1
|
20
20
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
21
|
-
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://
|
21
|
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
|
22
22
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
23
23
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
24
24
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
data/ext/README.md
CHANGED
data/ext/Rakefile
CHANGED
@@ -4,30 +4,14 @@ require "fileutils"
|
|
4
4
|
require "open-uri"
|
5
5
|
|
6
6
|
task :default => :clean do
|
7
|
-
# MiniPortile#download_file_http is a monkey patch that removes the download
|
8
|
-
# progress indicator. This indicator relies on the 'Content Length' response
|
9
|
-
# headers, which is not set by GitHub
|
10
|
-
class MiniPortile
|
11
|
-
def download_file_http(url, full_path, _count)
|
12
|
-
filename = File.basename(full_path)
|
13
|
-
with_tempfile(filename, full_path) do |temp_file|
|
14
|
-
params = { 'Accept-Encoding' => 'identity' }
|
15
|
-
OpenURI.open_uri(url, 'rb', params) do |io|
|
16
|
-
temp_file.write(io.read)
|
17
|
-
end
|
18
|
-
output
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
7
|
# Download and compile librdkafka
|
24
8
|
recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
|
25
9
|
|
26
10
|
# Use default homebrew openssl if we're on mac and the directory exists
|
27
11
|
# and each of flags is not empty
|
28
|
-
if recipe.host&.include?("darwin") && Dir.exist?("
|
29
|
-
ENV["CPPFLAGS"] = "-I/
|
30
|
-
ENV["LDFLAGS"] = "-L/
|
12
|
+
if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
|
13
|
+
ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
|
14
|
+
ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
|
31
15
|
end
|
32
16
|
|
33
17
|
recipe.files << {
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -498,11 +498,11 @@ module Rdkafka
|
|
498
498
|
# Exception behavior is more complicated than with `each`, in that if
|
499
499
|
# :yield_on_error is true, and an exception is raised during the
|
500
500
|
# poll, and messages have already been received, they will be yielded to
|
501
|
-
# the caller before the exception is allowed to
|
501
|
+
# the caller before the exception is allowed to propagate.
|
502
502
|
#
|
503
503
|
# If you are setting either auto.commit or auto.offset.store to false in
|
504
504
|
# the consumer configuration, then you should let yield_on_error keep its
|
505
|
-
# default value of false because you are
|
505
|
+
# default value of false because you are guaranteed to see these messages
|
506
506
|
# again. However, if both auto.commit and auto.offset.store are set to
|
507
507
|
# true, you should set yield_on_error to true so you can process messages
|
508
508
|
# that you may or may not see again.
|
@@ -518,7 +518,7 @@ module Rdkafka
|
|
518
518
|
# @yield [messages, pending_exception]
|
519
519
|
# @yieldparam messages [Array] An array of received Message
|
520
520
|
# @yieldparam pending_exception [Exception] normally nil, or an exception
|
521
|
-
# which will be
|
521
|
+
# which will be propagated after processing of the partial batch is complete.
|
522
522
|
#
|
523
523
|
# @return [nil]
|
524
524
|
def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -75,8 +75,9 @@ module Rdkafka
|
|
75
75
|
#
|
76
76
|
# @param topic [String] The topic to produce to
|
77
77
|
# @param payload [String,nil] The message's payload
|
78
|
-
# @param key [String] The message's key
|
78
|
+
# @param key [String, nil] The message's key
|
79
79
|
# @param partition [Integer,nil] Optional partition to produce to
|
80
|
+
# @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
|
80
81
|
# @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
|
81
82
|
# @param headers [Hash<String,String>] Optional message headers
|
82
83
|
#
|
data/lib/rdkafka/version.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Rdkafka
|
2
|
-
VERSION = "0.
|
3
|
-
LIBRDKAFKA_VERSION = "1.
|
4
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
2
|
+
VERSION = "0.11.1"
|
3
|
+
LIBRDKAFKA_VERSION = "1.8.2"
|
4
|
+
LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
|
5
5
|
end
|
data/rdkafka.gemspec
CHANGED
@@ -14,15 +14,17 @@ Gem::Specification.new do |gem|
|
|
14
14
|
gem.name = 'rdkafka'
|
15
15
|
gem.require_paths = ['lib']
|
16
16
|
gem.version = Rdkafka::VERSION
|
17
|
-
gem.required_ruby_version = '>= 2.
|
17
|
+
gem.required_ruby_version = '>= 2.6'
|
18
18
|
gem.extensions = %w(ext/Rakefile)
|
19
19
|
|
20
|
-
gem.add_dependency 'ffi', '~> 1.
|
21
|
-
gem.add_dependency 'mini_portile2', '~> 2.
|
22
|
-
gem.add_dependency 'rake', '
|
20
|
+
gem.add_dependency 'ffi', '~> 1.15'
|
21
|
+
gem.add_dependency 'mini_portile2', '~> 2.6'
|
22
|
+
gem.add_dependency 'rake', '> 12'
|
23
23
|
|
24
|
-
gem.add_development_dependency 'pry'
|
24
|
+
gem.add_development_dependency 'pry'
|
25
25
|
gem.add_development_dependency 'rspec', '~> 3.5'
|
26
|
-
gem.add_development_dependency 'rake'
|
27
|
-
gem.add_development_dependency 'simplecov'
|
26
|
+
gem.add_development_dependency 'rake'
|
27
|
+
gem.add_development_dependency 'simplecov'
|
28
|
+
gem.add_development_dependency 'guard'
|
29
|
+
gem.add_development_dependency 'guard-rspec'
|
28
30
|
end
|
data/spec/rdkafka/admin_spec.rb
CHANGED
data/spec/rdkafka/config_spec.rb
CHANGED
@@ -108,7 +108,7 @@ describe Rdkafka::Config do
|
|
108
108
|
end
|
109
109
|
|
110
110
|
it "should create a consumer with valid config" do
|
111
|
-
consumer =
|
111
|
+
consumer = rdkafka_consumer_config.consumer
|
112
112
|
expect(consumer).to be_a Rdkafka::Consumer
|
113
113
|
consumer.close
|
114
114
|
end
|
@@ -136,7 +136,7 @@ describe Rdkafka::Config do
|
|
136
136
|
end
|
137
137
|
|
138
138
|
it "should create a producer with valid config" do
|
139
|
-
producer =
|
139
|
+
producer = rdkafka_consumer_config.producer
|
140
140
|
expect(producer).to be_a Rdkafka::Producer
|
141
141
|
producer.close
|
142
142
|
end
|
@@ -3,9 +3,8 @@ require "ostruct"
|
|
3
3
|
require 'securerandom'
|
4
4
|
|
5
5
|
describe Rdkafka::Consumer do
|
6
|
-
let(:
|
7
|
-
let(:
|
8
|
-
let(:producer) { config.producer }
|
6
|
+
let(:consumer) { rdkafka_consumer_config.consumer }
|
7
|
+
let(:producer) { rdkafka_producer_config.producer }
|
9
8
|
|
10
9
|
after { consumer.close }
|
11
10
|
after { producer.close }
|
@@ -328,7 +327,7 @@ describe Rdkafka::Consumer do
|
|
328
327
|
before :all do
|
329
328
|
# Make sure there are some messages.
|
330
329
|
handles = []
|
331
|
-
producer =
|
330
|
+
producer = rdkafka_producer_config.producer
|
332
331
|
10.times do
|
333
332
|
(0..2).each do |i|
|
334
333
|
handles << producer.produce(
|
@@ -404,7 +403,7 @@ describe Rdkafka::Consumer do
|
|
404
403
|
config = {}
|
405
404
|
config[:'enable.auto.offset.store'] = false
|
406
405
|
config[:'enable.auto.commit'] = false
|
407
|
-
@new_consumer =
|
406
|
+
@new_consumer = rdkafka_consumer_config(config).consumer
|
408
407
|
@new_consumer.subscribe("consume_test_topic")
|
409
408
|
wait_for_assignment(@new_consumer)
|
410
409
|
end
|
@@ -459,13 +458,13 @@ describe Rdkafka::Consumer do
|
|
459
458
|
end
|
460
459
|
|
461
460
|
describe "#lag" do
|
462
|
-
let(:
|
461
|
+
let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
|
463
462
|
|
464
463
|
it "should calculate the consumer lag" do
|
465
464
|
# Make sure there's a message in every partition and
|
466
465
|
# wait for the message to make sure everything is committed.
|
467
466
|
(0..2).each do |i|
|
468
|
-
|
467
|
+
producer.produce(
|
469
468
|
topic: "consume_test_topic",
|
470
469
|
key: "key lag #{i}",
|
471
470
|
partition: i
|
@@ -508,7 +507,7 @@ describe Rdkafka::Consumer do
|
|
508
507
|
|
509
508
|
# Produce message on every topic again
|
510
509
|
(0..2).each do |i|
|
511
|
-
|
510
|
+
producer.produce(
|
512
511
|
topic: "consume_test_topic",
|
513
512
|
key: "key lag #{i}",
|
514
513
|
partition: i
|
@@ -824,8 +823,12 @@ describe Rdkafka::Consumer do
|
|
824
823
|
|
825
824
|
context "error raised from poll and yield_on_error is true" do
|
826
825
|
it "should yield buffered exceptions on rebalance, then break" do
|
827
|
-
config =
|
828
|
-
|
826
|
+
config = rdkafka_consumer_config(
|
827
|
+
{
|
828
|
+
:"enable.auto.commit" => false,
|
829
|
+
:"enable.auto.offset.store" => false
|
830
|
+
}
|
831
|
+
)
|
829
832
|
consumer = config.consumer
|
830
833
|
consumer.subscribe(topic_name)
|
831
834
|
loop_count = 0
|
@@ -864,8 +867,12 @@ describe Rdkafka::Consumer do
|
|
864
867
|
|
865
868
|
context "error raised from poll and yield_on_error is false" do
|
866
869
|
it "should yield buffered exceptions on rebalance, then break" do
|
867
|
-
config =
|
868
|
-
|
870
|
+
config = rdkafka_consumer_config(
|
871
|
+
{
|
872
|
+
:"enable.auto.commit" => false,
|
873
|
+
:"enable.auto.offset.store" => false
|
874
|
+
}
|
875
|
+
)
|
869
876
|
consumer = config.consumer
|
870
877
|
consumer.subscribe(topic_name)
|
871
878
|
loop_count = 0
|
@@ -902,51 +909,64 @@ describe Rdkafka::Consumer do
|
|
902
909
|
end
|
903
910
|
|
904
911
|
describe "a rebalance listener" do
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
912
|
+
let(:consumer) do
|
913
|
+
config = rdkafka_consumer_config
|
914
|
+
config.consumer_rebalance_listener = listener
|
915
|
+
config.consumer
|
916
|
+
end
|
910
917
|
|
911
|
-
|
912
|
-
|
913
|
-
|
918
|
+
context "with a working listener" do
|
919
|
+
let(:listener) do
|
920
|
+
Struct.new(:queue) do
|
921
|
+
def on_partitions_assigned(consumer, list)
|
922
|
+
collect(:assign, list)
|
923
|
+
end
|
914
924
|
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
925
|
+
def on_partitions_revoked(consumer, list)
|
926
|
+
collect(:revoke, list)
|
927
|
+
end
|
928
|
+
|
929
|
+
def collect(name, list)
|
930
|
+
partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
|
931
|
+
queue << ([name] + partitions)
|
932
|
+
end
|
933
|
+
end.new([])
|
934
|
+
end
|
920
935
|
|
921
|
-
|
936
|
+
it "should get notifications" do
|
937
|
+
notify_listener(listener)
|
922
938
|
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
939
|
+
expect(listener.queue).to eq([
|
940
|
+
[:assign, "consume_test_topic", 0, 1, 2],
|
941
|
+
[:revoke, "consume_test_topic", 0, 1, 2]
|
942
|
+
])
|
943
|
+
end
|
927
944
|
end
|
928
945
|
|
929
|
-
|
930
|
-
|
931
|
-
|
932
|
-
|
933
|
-
|
934
|
-
|
946
|
+
context "with a broken listener" do
|
947
|
+
let(:listener) do
|
948
|
+
Struct.new(:queue) do
|
949
|
+
def on_partitions_assigned(consumer, list)
|
950
|
+
queue << :assigned
|
951
|
+
raise 'boom'
|
952
|
+
end
|
935
953
|
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
|
954
|
+
def on_partitions_revoked(consumer, list)
|
955
|
+
queue << :revoked
|
956
|
+
raise 'boom'
|
957
|
+
end
|
958
|
+
end.new([])
|
959
|
+
end
|
941
960
|
|
942
|
-
|
961
|
+
it 'should handle callback exceptions' do
|
962
|
+
notify_listener(listener)
|
943
963
|
|
944
|
-
|
964
|
+
expect(listener.queue).to eq([:assigned, :revoked])
|
965
|
+
end
|
945
966
|
end
|
946
967
|
|
947
968
|
def notify_listener(listener)
|
948
969
|
# 1. subscribe and poll
|
949
|
-
config.consumer_rebalance_listener = listener
|
950
970
|
consumer.subscribe("consume_test_topic")
|
951
971
|
wait_for_assignment(consumer)
|
952
972
|
consumer.poll(100)
|
@@ -2,7 +2,7 @@ require "spec_helper"
|
|
2
2
|
require "securerandom"
|
3
3
|
|
4
4
|
describe Rdkafka::Metadata do
|
5
|
-
let(:config) {
|
5
|
+
let(:config) { rdkafka_consumer_config }
|
6
6
|
let(:native_config) { config.send(:native_config) }
|
7
7
|
let(:native_kafka) { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
|
8
8
|
|
@@ -1,8 +1,9 @@
|
|
1
1
|
require "spec_helper"
|
2
|
+
require "zlib"
|
2
3
|
|
3
4
|
describe Rdkafka::Producer do
|
4
|
-
let(:producer) {
|
5
|
-
let(:consumer) {
|
5
|
+
let(:producer) { rdkafka_producer_config.producer }
|
6
|
+
let(:consumer) { rdkafka_consumer_config.consumer }
|
6
7
|
|
7
8
|
after do
|
8
9
|
# Registry should always end up being empty
|
@@ -388,7 +389,7 @@ describe Rdkafka::Producer do
|
|
388
389
|
reader.close
|
389
390
|
|
390
391
|
# Avoids sharing the socket between processes.
|
391
|
-
producer =
|
392
|
+
producer = rdkafka_producer_config.producer
|
392
393
|
|
393
394
|
handle = producer.produce(
|
394
395
|
topic: "produce_test_topic",
|
data/spec/spec_helper.rb
CHANGED
@@ -8,27 +8,57 @@ end
|
|
8
8
|
require "pry"
|
9
9
|
require "rspec"
|
10
10
|
require "rdkafka"
|
11
|
+
require "timeout"
|
11
12
|
|
12
|
-
def
|
13
|
-
|
13
|
+
def rdkafka_base_config
|
14
|
+
{
|
14
15
|
:"api.version.request" => false,
|
15
16
|
:"broker.version.fallback" => "1.0",
|
16
17
|
:"bootstrap.servers" => "localhost:9092",
|
17
|
-
:"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
|
18
|
-
:"auto.offset.reset" => "earliest",
|
19
|
-
:"enable.partition.eof" => false
|
20
18
|
}
|
19
|
+
end
|
20
|
+
|
21
|
+
def rdkafka_config(config_overrides={})
|
22
|
+
# Generate the base config
|
23
|
+
config = rdkafka_base_config
|
24
|
+
# Merge overrides
|
25
|
+
config.merge!(config_overrides)
|
26
|
+
# Return it
|
27
|
+
Rdkafka::Config.new(config)
|
28
|
+
end
|
29
|
+
|
30
|
+
def rdkafka_consumer_config(config_overrides={})
|
31
|
+
# Generate the base config
|
32
|
+
config = rdkafka_base_config
|
33
|
+
# Add consumer specific fields to it
|
34
|
+
config[:"auto.offset.reset"] = "earliest"
|
35
|
+
config[:"enable.partition.eof"] = false
|
36
|
+
config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
|
37
|
+
# Enable debug mode if required
|
38
|
+
if ENV["DEBUG_CONSUMER"]
|
39
|
+
config[:debug] = "cgrp,topic,fetch"
|
40
|
+
end
|
41
|
+
# Merge overrides
|
42
|
+
config.merge!(config_overrides)
|
43
|
+
# Return it
|
44
|
+
Rdkafka::Config.new(config)
|
45
|
+
end
|
46
|
+
|
47
|
+
def rdkafka_producer_config(config_overrides={})
|
48
|
+
# Generate the base config
|
49
|
+
config = rdkafka_base_config
|
50
|
+
# Enable debug mode if required
|
21
51
|
if ENV["DEBUG_PRODUCER"]
|
22
52
|
config[:debug] = "broker,topic,msg"
|
23
|
-
elsif ENV["DEBUG_CONSUMER"]
|
24
|
-
config[:debug] = "cgrp,topic,fetch"
|
25
53
|
end
|
54
|
+
# Merge overrides
|
26
55
|
config.merge!(config_overrides)
|
56
|
+
# Return it
|
27
57
|
Rdkafka::Config.new(config)
|
28
58
|
end
|
29
59
|
|
30
60
|
def new_native_client
|
31
|
-
config =
|
61
|
+
config = rdkafka_consumer_config
|
32
62
|
config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
|
33
63
|
end
|
34
64
|
|
@@ -42,7 +72,7 @@ end
|
|
42
72
|
|
43
73
|
def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
|
44
74
|
new_consumer = !!consumer
|
45
|
-
consumer ||=
|
75
|
+
consumer ||= rdkafka_consumer_config.consumer
|
46
76
|
consumer.subscribe(topic)
|
47
77
|
timeout = Time.now.to_i + timeout_in_seconds
|
48
78
|
loop do
|
@@ -75,6 +105,9 @@ def wait_for_unassignment(consumer)
|
|
75
105
|
end
|
76
106
|
|
77
107
|
RSpec.configure do |config|
|
108
|
+
config.filter_run focus: true
|
109
|
+
config.run_all_when_everything_filtered = true
|
110
|
+
|
78
111
|
config.before(:suite) do
|
79
112
|
admin = rdkafka_config.admin
|
80
113
|
{
|
@@ -95,4 +128,12 @@ RSpec.configure do |config|
|
|
95
128
|
end
|
96
129
|
admin.close
|
97
130
|
end
|
131
|
+
|
132
|
+
config.around(:each) do |example|
|
133
|
+
# Timeout specs after a minute. If they take longer
|
134
|
+
# they are probably stuck
|
135
|
+
Timeout::timeout(60) do
|
136
|
+
example.run
|
137
|
+
end
|
138
|
+
end
|
98
139
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -16,56 +16,56 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.15'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.15'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: mini_portile2
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '2.
|
33
|
+
version: '2.6'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '2.
|
40
|
+
version: '2.6'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '12
|
47
|
+
version: '12'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '12
|
54
|
+
version: '12'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: pry
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0
|
61
|
+
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '0
|
68
|
+
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rspec
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,46 +84,78 @@ dependencies:
|
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: simplecov
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - "
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: '0
|
103
|
+
version: '0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - "
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: guard
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: guard-rspec
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
109
137
|
- !ruby/object:Gem::Version
|
110
|
-
version: '0
|
138
|
+
version: '0'
|
111
139
|
description: Modern Kafka client library for Ruby based on librdkafka
|
112
140
|
email:
|
113
141
|
- thijs@appsignal.com
|
114
|
-
executables:
|
142
|
+
executables:
|
143
|
+
- console
|
115
144
|
extensions:
|
116
145
|
- ext/Rakefile
|
117
146
|
extra_rdoc_files: []
|
118
147
|
files:
|
119
148
|
- ".gitignore"
|
149
|
+
- ".rspec"
|
120
150
|
- ".semaphore/semaphore.yml"
|
121
151
|
- ".yardopts"
|
122
152
|
- CHANGELOG.md
|
123
153
|
- Gemfile
|
154
|
+
- Guardfile
|
124
155
|
- LICENSE
|
125
156
|
- README.md
|
126
157
|
- Rakefile
|
158
|
+
- bin/console
|
127
159
|
- docker-compose.yml
|
128
160
|
- ext/README.md
|
129
161
|
- ext/Rakefile
|
@@ -180,7 +212,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
180
212
|
requirements:
|
181
213
|
- - ">="
|
182
214
|
- !ruby/object:Gem::Version
|
183
|
-
version: '2.
|
215
|
+
version: '2.6'
|
184
216
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
185
217
|
requirements:
|
186
218
|
- - ">="
|