karafka 1.4.13 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +3 -3
- data/.github/workflows/ci.yml +85 -30
- data/.ruby-version +1 -1
- data/CHANGELOG.md +268 -7
- data/CONTRIBUTING.md +10 -19
- data/Gemfile +6 -0
- data/Gemfile.lock +44 -87
- data/LICENSE +17 -0
- data/LICENSE-COMM +89 -0
- data/LICENSE-LGPL +165 -0
- data/README.md +44 -48
- data/bin/benchmarks +85 -0
- data/bin/create_token +22 -0
- data/bin/integrations +237 -0
- data/bin/karafka +4 -0
- data/bin/scenario +29 -0
- data/bin/stress_many +13 -0
- data/bin/stress_one +13 -0
- data/bin/wait_for_kafka +20 -0
- data/certs/karafka-pro.pem +11 -0
- data/config/errors.yml +55 -40
- data/docker-compose.yml +39 -3
- data/karafka.gemspec +11 -17
- data/lib/active_job/karafka.rb +21 -0
- data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
- data/lib/karafka/active_job/consumer.rb +26 -0
- data/lib/karafka/active_job/dispatcher.rb +38 -0
- data/lib/karafka/active_job/job_extensions.rb +34 -0
- data/lib/karafka/active_job/job_options_contract.rb +21 -0
- data/lib/karafka/active_job/routing/extensions.rb +31 -0
- data/lib/karafka/app.rb +15 -20
- data/lib/karafka/base_consumer.rb +181 -31
- data/lib/karafka/cli/base.rb +4 -4
- data/lib/karafka/cli/info.rb +43 -9
- data/lib/karafka/cli/install.rb +19 -10
- data/lib/karafka/cli/server.rb +17 -42
- data/lib/karafka/cli.rb +4 -11
- data/lib/karafka/connection/client.rb +385 -90
- data/lib/karafka/connection/listener.rb +246 -38
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +84 -0
- data/lib/karafka/connection/pauses_manager.rb +46 -0
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/connection/rebalance_manager.rb +78 -0
- data/lib/karafka/contracts/base.rb +17 -0
- data/lib/karafka/contracts/config.rb +88 -11
- data/lib/karafka/contracts/consumer_group.rb +21 -189
- data/lib/karafka/contracts/consumer_group_topic.rb +34 -11
- data/lib/karafka/contracts/server_cli_options.rb +19 -18
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/env.rb +46 -0
- data/lib/karafka/errors.rb +21 -21
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/helpers/colorize.rb +20 -0
- data/lib/karafka/helpers/multi_delegator.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
- data/lib/karafka/instrumentation/logger_listener.rb +164 -0
- data/lib/karafka/instrumentation/monitor.rb +13 -61
- data/lib/karafka/instrumentation/notifications.rb +52 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
- data/lib/karafka/instrumentation.rb +21 -0
- data/lib/karafka/licenser.rb +75 -0
- data/lib/karafka/messages/batch_metadata.rb +45 -0
- data/lib/karafka/messages/builders/batch_metadata.rb +40 -0
- data/lib/karafka/messages/builders/message.rb +39 -0
- data/lib/karafka/messages/builders/messages.rb +32 -0
- data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
- data/lib/karafka/messages/messages.rb +64 -0
- data/lib/karafka/{params → messages}/metadata.rb +4 -6
- data/lib/karafka/messages/seek.rb +9 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
- data/lib/karafka/pro/active_job/consumer.rb +46 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
- data/lib/karafka/pro/base_consumer.rb +82 -0
- data/lib/karafka/pro/contracts/base.rb +21 -0
- data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +33 -0
- data/lib/karafka/pro/loader.rb +76 -0
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/coordinator.rb +72 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/partitioner.rb +60 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
- data/lib/karafka/pro/routing/topic_extensions.rb +38 -0
- data/lib/karafka/pro.rb +13 -0
- data/lib/karafka/process.rb +1 -0
- data/lib/karafka/processing/coordinator.rb +88 -0
- data/lib/karafka/processing/coordinators_buffer.rb +54 -0
- data/lib/karafka/processing/executor.rb +118 -0
- data/lib/karafka/processing/executors_buffer.rb +88 -0
- data/lib/karafka/processing/jobs/base.rb +51 -0
- data/lib/karafka/processing/jobs/consume.rb +42 -0
- data/lib/karafka/processing/jobs/revoked.rb +22 -0
- data/lib/karafka/processing/jobs/shutdown.rb +23 -0
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/jobs_queue.rb +144 -0
- data/lib/karafka/processing/partitioner.rb +22 -0
- data/lib/karafka/processing/result.rb +29 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +88 -0
- data/lib/karafka/processing/workers_batch.rb +27 -0
- data/lib/karafka/railtie.rb +113 -0
- data/lib/karafka/routing/builder.rb +15 -24
- data/lib/karafka/routing/consumer_group.rb +11 -19
- data/lib/karafka/routing/consumer_mapper.rb +1 -2
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +53 -0
- data/lib/karafka/routing/subscription_groups_builder.rb +53 -0
- data/lib/karafka/routing/topic.rb +61 -24
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +51 -0
- data/lib/karafka/serialization/json/deserializer.rb +6 -15
- data/lib/karafka/server.rb +67 -26
- data/lib/karafka/setup/config.rb +147 -175
- data/lib/karafka/status.rb +14 -5
- data/lib/karafka/templates/example_consumer.rb.erb +16 -0
- data/lib/karafka/templates/karafka.rb.erb +15 -51
- data/lib/karafka/time_trackers/base.rb +19 -0
- data/lib/karafka/time_trackers/pause.rb +92 -0
- data/lib/karafka/time_trackers/poll.rb +65 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +38 -17
- data.tar.gz.sig +0 -0
- metadata +118 -120
- metadata.gz.sig +0 -0
- data/MIT-LICENCE +0 -18
- data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
- data/lib/karafka/attributes_map.rb +0 -63
- data/lib/karafka/backends/inline.rb +0 -16
- data/lib/karafka/base_responder.rb +0 -226
- data/lib/karafka/cli/flow.rb +0 -48
- data/lib/karafka/cli/missingno.rb +0 -19
- data/lib/karafka/code_reloader.rb +0 -67
- data/lib/karafka/connection/api_adapter.rb +0 -158
- data/lib/karafka/connection/batch_delegator.rb +0 -55
- data/lib/karafka/connection/builder.rb +0 -23
- data/lib/karafka/connection/message_delegator.rb +0 -36
- data/lib/karafka/consumers/batch_metadata.rb +0 -10
- data/lib/karafka/consumers/callbacks.rb +0 -71
- data/lib/karafka/consumers/includer.rb +0 -64
- data/lib/karafka/consumers/responders.rb +0 -24
- data/lib/karafka/consumers/single_params.rb +0 -15
- data/lib/karafka/contracts/responder_usage.rb +0 -54
- data/lib/karafka/fetcher.rb +0 -42
- data/lib/karafka/helpers/class_matcher.rb +0 -88
- data/lib/karafka/helpers/config_retriever.rb +0 -46
- data/lib/karafka/helpers/inflector.rb +0 -26
- data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
- data/lib/karafka/params/batch_metadata.rb +0 -26
- data/lib/karafka/params/builders/batch_metadata.rb +0 -30
- data/lib/karafka/params/builders/params.rb +0 -38
- data/lib/karafka/params/builders/params_batch.rb +0 -25
- data/lib/karafka/params/params_batch.rb +0 -60
- data/lib/karafka/patches/ruby_kafka.rb +0 -47
- data/lib/karafka/persistence/client.rb +0 -29
- data/lib/karafka/persistence/consumers.rb +0 -45
- data/lib/karafka/persistence/topics.rb +0 -48
- data/lib/karafka/responders/builder.rb +0 -36
- data/lib/karafka/responders/topic.rb +0 -55
- data/lib/karafka/routing/topic_mapper.rb +0 -53
- data/lib/karafka/serialization/json/serializer.rb +0 -31
- data/lib/karafka/setup/configurators/water_drop.rb +0 -36
- data/lib/karafka/templates/application_responder.rb.erb +0 -11
data/bin/benchmarks
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Runner for running given benchmark cases
|
4
|
+
# Some of the cases require pre-populated data and we populate this in places that need it
|
5
|
+
# In other cases we generate this data in a background process, so the partitions data stream
|
6
|
+
# is consistent and we don't end up consuming huge batches of a single partition.
|
7
|
+
|
8
|
+
require 'open3'
|
9
|
+
require 'pathname'
|
10
|
+
|
11
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
|
13
|
+
|
14
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
15
|
+
|
16
|
+
# Load all the benchmarks
|
17
|
+
benchmarks = Dir[ROOT_PATH.join('spec/benchmarks/**/*.rb')]
|
18
|
+
|
19
|
+
# If filter is provided, apply
|
20
|
+
benchmarks.delete_if { |name| !name.include?(ARGV[0]) } if ARGV[0]
|
21
|
+
|
22
|
+
raise ArgumentError, "No benchmarks with filter: #{ARGV[0]}" if benchmarks.empty?
|
23
|
+
|
24
|
+
# We may skip seeding if we are running the benchmarks multiple times, then since we do not
|
25
|
+
# commit offsets we can skip generating more data
|
26
|
+
if ENV['SEED']
|
27
|
+
require 'spec/benchmarks_helper'
|
28
|
+
|
29
|
+
# We need to setup karafka here to have producer for data seeding
|
30
|
+
setup_karafka
|
31
|
+
|
32
|
+
# This takes some time but needs to run only once per benchmark session
|
33
|
+
puts 'Seeding benchmarks data...'
|
34
|
+
|
35
|
+
producer = Karafka::App.producer
|
36
|
+
|
37
|
+
# We make our data json compatible so we can also benchmark serialization
|
38
|
+
elements = Array.new(100_000) { { a: :b }.to_json }
|
39
|
+
|
40
|
+
# We do not populate data of benchmarks_0_10 as we use it with life-stream data only
|
41
|
+
%w[
|
42
|
+
benchmarks_00_01
|
43
|
+
benchmarks_00_05
|
44
|
+
].each do |topic_name|
|
45
|
+
partitions_count = topic_name.split('_').last.to_i
|
46
|
+
|
47
|
+
partitions_count.times do |partition|
|
48
|
+
puts "Seeding #{topic_name}:#{partition}"
|
49
|
+
|
50
|
+
elements.each_slice(10_000) do |data_slice|
|
51
|
+
data = data_slice.map do |data|
|
52
|
+
{ topic: topic_name, payload: data, partition: partition }
|
53
|
+
end
|
54
|
+
|
55
|
+
producer.buffer_many(data)
|
56
|
+
producer.flush_sync
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Selects requested benchmarks and runs them one after another
|
63
|
+
benchmarks.each do |benchmark_path|
|
64
|
+
puts "Running #{benchmark_path.gsub("#{ROOT_PATH}/spec/benchmarks/", '')}"
|
65
|
+
|
66
|
+
benchmark = "bundle exec ruby -r ./spec/benchmarks_helper.rb #{benchmark_path}"
|
67
|
+
|
68
|
+
Open3.popen3(benchmark) do |stdin, stdout, stderr, thread|
|
69
|
+
t1 = Thread.new do
|
70
|
+
while line = stdout.gets
|
71
|
+
puts(line)
|
72
|
+
end
|
73
|
+
rescue IOError
|
74
|
+
end
|
75
|
+
|
76
|
+
t2 = Thread.new do
|
77
|
+
while line = stderr.gets
|
78
|
+
puts(line)
|
79
|
+
end
|
80
|
+
rescue IOError
|
81
|
+
end
|
82
|
+
|
83
|
+
thread.join
|
84
|
+
end
|
85
|
+
end
|
data/bin/create_token
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'openssl'
|
4
|
+
require 'base64'
|
5
|
+
require 'json'
|
6
|
+
require 'date'
|
7
|
+
|
8
|
+
PRIVATE_KEY_LOCATION = File.join(Dir.home, '.ssh', 'karafka-pro', 'id_rsa')
|
9
|
+
|
10
|
+
# Name of the entity that acquires the license
|
11
|
+
ENTITY = ARGV[0]
|
12
|
+
|
13
|
+
raise ArgumentError, 'Entity missing' if ENTITY.nil? || ENTITY.empty?
|
14
|
+
|
15
|
+
pro_token_data = { entity: ENTITY }
|
16
|
+
|
17
|
+
# This code uses my private key to generate a new token for Karafka Pro capabilities
|
18
|
+
private_key = OpenSSL::PKey::RSA.new(File.read(PRIVATE_KEY_LOCATION))
|
19
|
+
|
20
|
+
bin_key = private_key.private_encrypt(pro_token_data.to_json)
|
21
|
+
|
22
|
+
puts Base64.encode64(bin_key)
|
data/bin/integrations
ADDED
@@ -0,0 +1,237 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Runner to run integration specs in parallel
|
4
|
+
|
5
|
+
# Part of integration specs run pristine without bundler.
|
6
|
+
# If we would run bundle exec when running this code, bundler would inject its own context
|
7
|
+
# into them, messing things up heavily
|
8
|
+
raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
|
9
|
+
|
10
|
+
require 'open3'
|
11
|
+
require 'fileutils'
|
12
|
+
require 'pathname'
|
13
|
+
require 'tmpdir'
|
14
|
+
require 'etc'
|
15
|
+
|
16
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
17
|
+
|
18
|
+
# How many child processes with integration specs do we want to run in parallel
|
19
|
+
# When the value is high, there's a problem with thread allocation on Github CI, tht is why
|
20
|
+
# we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
|
21
|
+
# of CPU
|
22
|
+
CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
|
23
|
+
|
24
|
+
# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
|
25
|
+
MAX_BUFFER_OUTPUT = 51_200
|
26
|
+
|
27
|
+
# Abstraction around a single test scenario execution process
|
28
|
+
class Scenario
|
29
|
+
# How long a scenario can run before we kill it
|
30
|
+
# This is a fail-safe just in case something would hang
|
31
|
+
MAX_RUN_TIME = 3 * 60 # 3 minutes tops
|
32
|
+
|
33
|
+
# There are rare cases where Karafka may force shutdown for some of the integration cases
|
34
|
+
# This includes exactly those
|
35
|
+
EXIT_CODES = {
|
36
|
+
default: [0],
|
37
|
+
'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
|
38
|
+
'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
|
39
|
+
'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
|
40
|
+
'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
|
41
|
+
}.freeze
|
42
|
+
|
43
|
+
private_constant :MAX_RUN_TIME, :EXIT_CODES
|
44
|
+
|
45
|
+
# Creates scenario instance and runs in the background process
|
46
|
+
#
|
47
|
+
# @param path [String] path to the scenarios file
|
48
|
+
def initialize(path)
|
49
|
+
@path = path
|
50
|
+
# Last 1024 characters from stdout
|
51
|
+
@stdout_tail = ''
|
52
|
+
end
|
53
|
+
|
54
|
+
# Starts running given scenario in a separate process
|
55
|
+
def start
|
56
|
+
@stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
|
57
|
+
@started_at = current_time
|
58
|
+
end
|
59
|
+
|
60
|
+
# @return [String] integration spec name
|
61
|
+
def name
|
62
|
+
@path.gsub("#{ROOT_PATH}/spec/integrations/", '')
|
63
|
+
end
|
64
|
+
|
65
|
+
# @return [Boolean] true if spec is pristine
|
66
|
+
def pristine?
|
67
|
+
scenario_dir = File.dirname(@path)
|
68
|
+
|
69
|
+
# If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
|
70
|
+
# to run bundle install, etc in order to run it
|
71
|
+
File.exist?(File.join(scenario_dir, 'Gemfile'))
|
72
|
+
end
|
73
|
+
|
74
|
+
# @return [Boolean] did this scenario finished or is it still running
|
75
|
+
def finished?
|
76
|
+
# If the thread is running too long, kill it
|
77
|
+
if current_time - @started_at > MAX_RUN_TIME
|
78
|
+
@wait_thr.kill
|
79
|
+
|
80
|
+
begin
|
81
|
+
Process.kill('TERM', pid)
|
82
|
+
# It may finish right after we want to kill it, that's why we ignore this
|
83
|
+
rescue Errno::ESRCH
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# We read it so it won't grow as we use our default logger that prints to both test.log and
|
88
|
+
# to stdout. Otherwise after reaching the buffer size, it would hang
|
89
|
+
buffer = ''
|
90
|
+
@stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
|
91
|
+
@stdout_tail << buffer
|
92
|
+
@stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
|
93
|
+
|
94
|
+
!@wait_thr.alive?
|
95
|
+
end
|
96
|
+
|
97
|
+
# @return [Boolean] did this scenario finish successfully or not
|
98
|
+
def success?
|
99
|
+
expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
|
100
|
+
|
101
|
+
expected_exit_codes.include?(exit_code)
|
102
|
+
end
|
103
|
+
|
104
|
+
# @return [Integer] pid of the process of this scenario
|
105
|
+
def pid
|
106
|
+
@wait_thr.pid
|
107
|
+
end
|
108
|
+
|
109
|
+
# @return [Integer] exit code of the process running given scenario
|
110
|
+
def exit_code
|
111
|
+
# There may be no exit status if we killed the thread
|
112
|
+
@wait_thr.value&.exitstatus || 123
|
113
|
+
end
|
114
|
+
|
115
|
+
# Prints a status report when scenario is finished and stdout if it failed
|
116
|
+
def report
|
117
|
+
if success?
|
118
|
+
print "\e[#{32}m#{'.'}\e[0m"
|
119
|
+
else
|
120
|
+
buffer = ''
|
121
|
+
|
122
|
+
@stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
|
123
|
+
|
124
|
+
puts
|
125
|
+
puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
|
126
|
+
puts "Exit code: #{exit_code}"
|
127
|
+
puts @stdout_tail
|
128
|
+
puts buffer
|
129
|
+
puts
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
private
|
134
|
+
|
135
|
+
# Sets up a proper environment for a given spec to run and returns the run command
|
136
|
+
# @return [String] run command
|
137
|
+
def init_and_build_cmd
|
138
|
+
# If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
|
139
|
+
# to run bundle install, etc in order to run it
|
140
|
+
if pristine?
|
141
|
+
scenario_dir = File.dirname(@path)
|
142
|
+
# We copy the spec into a temp dir, not to pollute the spec location with logs, etc
|
143
|
+
temp_dir = Dir.mktmpdir
|
144
|
+
file_name = File.basename(@path)
|
145
|
+
|
146
|
+
FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
|
147
|
+
|
148
|
+
<<~CMD
|
149
|
+
cd #{temp_dir} &&
|
150
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
151
|
+
BUNDLE_AUTO_INSTALL=true \
|
152
|
+
PRISTINE_MODE=true \
|
153
|
+
bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{file_name}
|
154
|
+
CMD
|
155
|
+
else
|
156
|
+
<<~CMD
|
157
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
158
|
+
bundle exec ruby -r ./spec/integrations_helper.rb #{@path}
|
159
|
+
CMD
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
# @return [Float] current machine time
|
164
|
+
def current_time
|
165
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
# Load all the specs
|
170
|
+
specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
|
171
|
+
|
172
|
+
# If filters is provided, apply
|
173
|
+
# Allows to provide several filters one after another and applies all of them
|
174
|
+
ARGV.each do |filter|
|
175
|
+
specs.delete_if { |name| !name.include?(filter) }
|
176
|
+
end
|
177
|
+
|
178
|
+
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
179
|
+
|
180
|
+
# Randomize order
|
181
|
+
seed = (ENV['SEED'] || rand(0..10_000)).to_i
|
182
|
+
|
183
|
+
puts "Random seed: #{seed}"
|
184
|
+
|
185
|
+
scenarios = specs
|
186
|
+
.shuffle(random: Random.new(seed))
|
187
|
+
.map { |integration_test| Scenario.new(integration_test) }
|
188
|
+
|
189
|
+
regulars = scenarios.reject(&:pristine?)
|
190
|
+
pristine = scenarios.select(&:pristine?)
|
191
|
+
|
192
|
+
active_scenarios = []
|
193
|
+
finished_scenarios = []
|
194
|
+
|
195
|
+
while finished_scenarios.size < scenarios.size
|
196
|
+
# If we have space to run another scenario, we add it
|
197
|
+
if active_scenarios.size < CONCURRENCY
|
198
|
+
scenario = nil
|
199
|
+
# We can run only one pristine at the same time due to concurrency issues within bundler
|
200
|
+
# Since they usually take longer than others, we try to run them as fast as possible when there
|
201
|
+
# is a slot
|
202
|
+
scenario = pristine.pop unless active_scenarios.any?(&:pristine?)
|
203
|
+
scenario ||= regulars.pop
|
204
|
+
|
205
|
+
if scenario
|
206
|
+
scenario.start
|
207
|
+
active_scenarios << scenario
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
active_scenarios.select(&:finished?).each do |exited|
|
212
|
+
scenario = active_scenarios.delete(exited)
|
213
|
+
scenario.report
|
214
|
+
finished_scenarios << scenario
|
215
|
+
end
|
216
|
+
|
217
|
+
sleep(0.1)
|
218
|
+
end
|
219
|
+
|
220
|
+
failed_scenarios = finished_scenarios.reject(&:success?)
|
221
|
+
|
222
|
+
# Report once more on the failed jobs
|
223
|
+
# This will only list scenarios that failed without printing their stdout here.
|
224
|
+
if failed_scenarios.empty?
|
225
|
+
puts
|
226
|
+
else
|
227
|
+
puts "\nFailed scenarios:\n\n"
|
228
|
+
|
229
|
+
failed_scenarios.each do |scenario|
|
230
|
+
puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
|
231
|
+
end
|
232
|
+
|
233
|
+
puts
|
234
|
+
|
235
|
+
# Exit with 1 if not all scenarios were successful
|
236
|
+
exit 1
|
237
|
+
end
|
data/bin/karafka
CHANGED
@@ -2,6 +2,10 @@
|
|
2
2
|
|
3
3
|
require 'karafka'
|
4
4
|
|
5
|
+
# We set this to indicate, that the process in which we are (whatever it does) was started using
|
6
|
+
# our bin/karafka cli
|
7
|
+
ENV['KARAFKA_CLI'] = 'true'
|
8
|
+
|
5
9
|
# If there is a boot file, we need to require it as we expect it to contain
|
6
10
|
# Karafka app setup, routes, etc
|
7
11
|
if File.exist?(Karafka.boot_file)
|
data/bin/scenario
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Runner for non-parallel execution of a single scenario.
|
4
|
+
# It prints all the info stdout, etc and basically replaces itself with the scenario execution.
|
5
|
+
# It is useful when we work with a single spec and we need all the debug info
|
6
|
+
|
7
|
+
raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
|
8
|
+
|
9
|
+
require 'open3'
|
10
|
+
require 'fileutils'
|
11
|
+
require 'pathname'
|
12
|
+
require 'tmpdir'
|
13
|
+
require 'etc'
|
14
|
+
|
15
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
16
|
+
|
17
|
+
# Load all the specs
|
18
|
+
specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
|
19
|
+
|
20
|
+
# If filters is provided, apply
|
21
|
+
# Allows to provide several filters one after another and applies all of them
|
22
|
+
ARGV.each do |filter|
|
23
|
+
specs.delete_if { |name| !name.include?(filter) }
|
24
|
+
end
|
25
|
+
|
26
|
+
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
27
|
+
raise ArgumentError, "Many specs found with filters: #{ARGV.join(', ')}" if specs.size != 1
|
28
|
+
|
29
|
+
exec("bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{specs[0]}")
|
data/bin/stress_many
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Runs integration specs in an endless loop
|
4
|
+
# This allows us to ensure (after long enough time) that the integrations test suit is stable and
|
5
|
+
# that there are no anomalies when running it for a long period of time
|
6
|
+
|
7
|
+
set -e
|
8
|
+
|
9
|
+
while :
|
10
|
+
do
|
11
|
+
clear
|
12
|
+
bin/integrations $1
|
13
|
+
done
|
data/bin/stress_one
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Runs a single integration spec in an endless loop
|
4
|
+
# This allows us to ensure (after long enough time) that the integration spec is stable and
|
5
|
+
# that there are no anomalies when running it for a long period of time
|
6
|
+
|
7
|
+
set -e
|
8
|
+
|
9
|
+
while :
|
10
|
+
do
|
11
|
+
clear
|
12
|
+
bin/scenario $1
|
13
|
+
done
|
data/bin/wait_for_kafka
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# This script allows us to wait for Kafka docker to fully be ready
|
4
|
+
# We consider it fully ready when all our topics that need to be created are created as expected
|
5
|
+
|
6
|
+
KAFKA_NAME='karafka_20_kafka'
|
7
|
+
ZOOKEEPER='zookeeper:2181'
|
8
|
+
LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
|
9
|
+
|
10
|
+
# Take the number of topics that we need to create prior to running anything
|
11
|
+
TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
|
12
|
+
|
13
|
+
# And wait until all of them are created
|
14
|
+
until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
|
15
|
+
do
|
16
|
+
echo "Waiting for Kafka to create all the needed topics..."
|
17
|
+
sleep 1
|
18
|
+
done
|
19
|
+
|
20
|
+
echo "All the needed topics created."
|
@@ -0,0 +1,11 @@
|
|
1
|
+
-----BEGIN RSA PUBLIC KEY-----
|
2
|
+
MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
|
3
|
+
tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
|
4
|
+
ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
|
5
|
+
Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
|
6
|
+
8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
|
7
|
+
BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
|
8
|
+
2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
|
9
|
+
2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
|
10
|
+
aQtjlQWKPx5xAgMBAAE=
|
11
|
+
-----END RSA PUBLIC KEY-----
|
data/config/errors.yml
CHANGED
@@ -1,41 +1,56 @@
|
|
1
1
|
en:
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
2
|
+
validations:
|
3
|
+
config:
|
4
|
+
missing: needs to be present
|
5
|
+
client_id_format: 'needs to be a string with a Kafka accepted format'
|
6
|
+
license.entity_format: needs to be a string
|
7
|
+
license.token_format: needs to be either false or a string
|
8
|
+
license.expires_on_format: needs to be a valid date
|
9
|
+
concurrency_format: needs to be an integer bigger than 0
|
10
|
+
consumer_mapper_format: needs to be present
|
11
|
+
consumer_persistence_format: needs to be either true or false
|
12
|
+
pause_timeout_format: needs to be an integer bigger than 0
|
13
|
+
pause_max_timeout_format: needs to be an integer bigger than 0
|
14
|
+
pause_with_exponential_backoff_format: needs to be either true or false
|
15
|
+
shutdown_timeout_format: needs to be an integer bigger than 0
|
16
|
+
max_wait_time_format: needs to be an integer bigger than 0
|
17
|
+
kafka_format: needs to be a filled hash
|
18
|
+
internal.status_format: needs to be present
|
19
|
+
internal.process_format: needs to be present
|
20
|
+
internal.routing.builder_format: needs to be present
|
21
|
+
internal.routing.subscription_groups_builder_format: needs to be present
|
22
|
+
key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
|
23
|
+
max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
|
24
|
+
shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
|
25
|
+
|
26
|
+
server_cli_options:
|
27
|
+
missing: needs to be present
|
28
|
+
consumer_groups_inclusion: Unknown consumer group
|
29
|
+
|
30
|
+
consumer_group_topic:
|
31
|
+
missing: needs to be present
|
32
|
+
name_format: 'needs to be a string with a Kafka accepted format'
|
33
|
+
deserializer_format: needs to be present
|
34
|
+
manual_offset_management_format: needs to be either true or false
|
35
|
+
consumer_format: needs to be present
|
36
|
+
id_format: 'needs to be a string with a Kafka accepted format'
|
37
|
+
initial_offset_format: needs to be either earliest or latest
|
38
|
+
|
39
|
+
consumer_group:
|
40
|
+
missing: needs to be present
|
41
|
+
topics_names_not_unique: all topic names within a single consumer group must be unique
|
42
|
+
id_format: 'needs to be a string with a Kafka accepted format'
|
43
|
+
topics_format: needs to be a non-empty array
|
44
|
+
|
45
|
+
job_options:
|
46
|
+
missing: needs to be present
|
47
|
+
dispatch_method_format: needs to be either :produce_async or :produce_sync
|
48
|
+
partitioner_format: 'needs to respond to #call'
|
49
|
+
partition_key_type_format: 'needs to be either :key or :partition_key'
|
50
|
+
|
51
|
+
test:
|
52
|
+
missing: needs to be present
|
53
|
+
id_format: needs to be a String
|
54
|
+
|
55
|
+
pro_consumer_group_topic:
|
56
|
+
consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
|
data/docker-compose.yml
CHANGED
@@ -1,17 +1,53 @@
|
|
1
1
|
version: '2'
|
2
2
|
services:
|
3
3
|
zookeeper:
|
4
|
+
container_name: karafka_20_zookeeper
|
4
5
|
image: wurstmeister/zookeeper
|
5
6
|
ports:
|
6
|
-
-
|
7
|
+
- '2181:2181'
|
7
8
|
kafka:
|
8
|
-
|
9
|
+
container_name: karafka_20_kafka
|
10
|
+
image: wurstmeister/kafka
|
9
11
|
ports:
|
10
|
-
-
|
12
|
+
- '9092:9092'
|
11
13
|
environment:
|
12
14
|
KAFKA_ADVERTISED_HOST_NAME: localhost
|
13
15
|
KAFKA_ADVERTISED_PORT: 9092
|
14
16
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
15
17
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
18
|
+
KAFKA_CREATE_TOPICS:
|
19
|
+
"integrations_00_02:2:1,\
|
20
|
+
integrations_01_02:2:1,\
|
21
|
+
integrations_02_02:2:1,\
|
22
|
+
integrations_03_02:2:1,\
|
23
|
+
integrations_04_02:2:1,\
|
24
|
+
integrations_05_02:2:1,\
|
25
|
+
integrations_06_02:2:1,\
|
26
|
+
integrations_07_02:2:1,\
|
27
|
+
integrations_08_02:2:1,\
|
28
|
+
integrations_09_02:2:1,\
|
29
|
+
integrations_10_02:2:1,\
|
30
|
+
integrations_11_02:2:1,\
|
31
|
+
integrations_12_02:2:1,\
|
32
|
+
integrations_13_02:2:1,\
|
33
|
+
integrations_14_02:2:1,\
|
34
|
+
integrations_15_02:2:1,\
|
35
|
+
integrations_16_02:2:1,\
|
36
|
+
integrations_17_02:2:1,\
|
37
|
+
integrations_18_02:2:1,\
|
38
|
+
integrations_19_02:2:1,\
|
39
|
+
integrations_20_02:2:1,\
|
40
|
+
integrations_21_02:2:1,\
|
41
|
+
integrations_00_03:3:1,\
|
42
|
+
integrations_01_03:3:1,\
|
43
|
+
integrations_02_03:3:1,\
|
44
|
+
integrations_03_03:3:1,\
|
45
|
+
integrations_04_03:3:1,\
|
46
|
+
integrations_00_10:10:1,\
|
47
|
+
integrations_01_10:10:1,\
|
48
|
+
benchmarks_00_01:1:1,\
|
49
|
+
benchmarks_00_05:5:1,\
|
50
|
+
benchmarks_01_05:5:1,\
|
51
|
+
benchmarks_00_10:10:1"
|
16
52
|
volumes:
|
17
53
|
- /var/run/docker.sock:/var/run/docker.sock
|
data/karafka.gemspec
CHANGED
@@ -5,29 +5,24 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
require 'karafka/version'
|
7
7
|
|
8
|
-
# rubocop:disable Metrics/BlockLength
|
9
8
|
Gem::Specification.new do |spec|
|
10
9
|
spec.name = 'karafka'
|
11
10
|
spec.version = ::Karafka::VERSION
|
12
11
|
spec.platform = Gem::Platform::RUBY
|
13
|
-
spec.authors = ['Maciej Mensfeld'
|
14
|
-
spec.email = %w[maciej@mensfeld.pl
|
12
|
+
spec.authors = ['Maciej Mensfeld']
|
13
|
+
spec.email = %w[maciej@mensfeld.pl]
|
15
14
|
spec.homepage = 'https://karafka.io'
|
16
|
-
spec.summary = '
|
15
|
+
spec.summary = 'Efficient Kafka processing framework for Ruby and Rails'
|
17
16
|
spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
|
18
|
-
spec.
|
17
|
+
spec.licenses = ['LGPL-3.0', 'Commercial']
|
19
18
|
|
20
|
-
spec.add_dependency '
|
21
|
-
spec.add_dependency '
|
22
|
-
spec.add_dependency '
|
23
|
-
spec.add_dependency '
|
24
|
-
spec.add_dependency '
|
25
|
-
spec.add_dependency 'ruby-kafka', '>= 1.3.0'
|
26
|
-
spec.add_dependency 'thor', '>= 1.1'
|
27
|
-
spec.add_dependency 'waterdrop', '~> 1.4'
|
28
|
-
spec.add_dependency 'zeitwerk', '~> 2.4'
|
19
|
+
spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
|
20
|
+
spec.add_dependency 'rdkafka', '>= 0.12'
|
21
|
+
spec.add_dependency 'thor', '>= 0.20'
|
22
|
+
spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
|
23
|
+
spec.add_dependency 'zeitwerk', '~> 2.3'
|
29
24
|
|
30
|
-
spec.required_ruby_version = '>= 2.7'
|
25
|
+
spec.required_ruby_version = '>= 2.7.0'
|
31
26
|
|
32
27
|
if $PROGRAM_NAME.end_with?('gem')
|
33
28
|
spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
|
@@ -35,7 +30,7 @@ Gem::Specification.new do |spec|
|
|
35
30
|
|
36
31
|
spec.cert_chain = %w[certs/mensfeld.pem]
|
37
32
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
|
38
|
-
spec.executables =
|
33
|
+
spec.executables = %w[karafka]
|
39
34
|
spec.require_paths = %w[lib]
|
40
35
|
|
41
36
|
spec.metadata = {
|
@@ -43,4 +38,3 @@ Gem::Specification.new do |spec|
|
|
43
38
|
'rubygems_mfa_required' => 'true'
|
44
39
|
}
|
45
40
|
end
|
46
|
-
# rubocop:enable Metrics/BlockLength
|