karafka 1.4.13 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +3 -3
- data/.github/workflows/ci.yml +85 -30
- data/.ruby-version +1 -1
- data/CHANGELOG.md +268 -7
- data/CONTRIBUTING.md +10 -19
- data/Gemfile +6 -0
- data/Gemfile.lock +44 -87
- data/LICENSE +17 -0
- data/LICENSE-COMM +89 -0
- data/LICENSE-LGPL +165 -0
- data/README.md +44 -48
- data/bin/benchmarks +85 -0
- data/bin/create_token +22 -0
- data/bin/integrations +237 -0
- data/bin/karafka +4 -0
- data/bin/scenario +29 -0
- data/bin/stress_many +13 -0
- data/bin/stress_one +13 -0
- data/bin/wait_for_kafka +20 -0
- data/certs/karafka-pro.pem +11 -0
- data/config/errors.yml +55 -40
- data/docker-compose.yml +39 -3
- data/karafka.gemspec +11 -17
- data/lib/active_job/karafka.rb +21 -0
- data/lib/active_job/queue_adapters/karafka_adapter.rb +26 -0
- data/lib/karafka/active_job/consumer.rb +26 -0
- data/lib/karafka/active_job/dispatcher.rb +38 -0
- data/lib/karafka/active_job/job_extensions.rb +34 -0
- data/lib/karafka/active_job/job_options_contract.rb +21 -0
- data/lib/karafka/active_job/routing/extensions.rb +31 -0
- data/lib/karafka/app.rb +15 -20
- data/lib/karafka/base_consumer.rb +181 -31
- data/lib/karafka/cli/base.rb +4 -4
- data/lib/karafka/cli/info.rb +43 -9
- data/lib/karafka/cli/install.rb +19 -10
- data/lib/karafka/cli/server.rb +17 -42
- data/lib/karafka/cli.rb +4 -11
- data/lib/karafka/connection/client.rb +385 -90
- data/lib/karafka/connection/listener.rb +246 -38
- data/lib/karafka/connection/listeners_batch.rb +24 -0
- data/lib/karafka/connection/messages_buffer.rb +84 -0
- data/lib/karafka/connection/pauses_manager.rb +46 -0
- data/lib/karafka/connection/raw_messages_buffer.rb +101 -0
- data/lib/karafka/connection/rebalance_manager.rb +78 -0
- data/lib/karafka/contracts/base.rb +17 -0
- data/lib/karafka/contracts/config.rb +88 -11
- data/lib/karafka/contracts/consumer_group.rb +21 -189
- data/lib/karafka/contracts/consumer_group_topic.rb +34 -11
- data/lib/karafka/contracts/server_cli_options.rb +19 -18
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/env.rb +46 -0
- data/lib/karafka/errors.rb +21 -21
- data/lib/karafka/helpers/async.rb +33 -0
- data/lib/karafka/helpers/colorize.rb +20 -0
- data/lib/karafka/helpers/multi_delegator.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
- data/lib/karafka/instrumentation/callbacks/statistics.rb +41 -0
- data/lib/karafka/instrumentation/logger_listener.rb +164 -0
- data/lib/karafka/instrumentation/monitor.rb +13 -61
- data/lib/karafka/instrumentation/notifications.rb +52 -0
- data/lib/karafka/instrumentation/proctitle_listener.rb +3 -3
- data/lib/karafka/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +232 -0
- data/lib/karafka/instrumentation.rb +21 -0
- data/lib/karafka/licenser.rb +75 -0
- data/lib/karafka/messages/batch_metadata.rb +45 -0
- data/lib/karafka/messages/builders/batch_metadata.rb +40 -0
- data/lib/karafka/messages/builders/message.rb +39 -0
- data/lib/karafka/messages/builders/messages.rb +32 -0
- data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
- data/lib/karafka/messages/messages.rb +64 -0
- data/lib/karafka/{params → messages}/metadata.rb +4 -6
- data/lib/karafka/messages/seek.rb +9 -0
- data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
- data/lib/karafka/pro/active_job/consumer.rb +46 -0
- data/lib/karafka/pro/active_job/dispatcher.rb +61 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +32 -0
- data/lib/karafka/pro/base_consumer.rb +82 -0
- data/lib/karafka/pro/contracts/base.rb +21 -0
- data/lib/karafka/pro/contracts/consumer_group.rb +34 -0
- data/lib/karafka/pro/contracts/consumer_group_topic.rb +33 -0
- data/lib/karafka/pro/loader.rb +76 -0
- data/lib/karafka/pro/performance_tracker.rb +80 -0
- data/lib/karafka/pro/processing/coordinator.rb +72 -0
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +37 -0
- data/lib/karafka/pro/processing/jobs_builder.rb +32 -0
- data/lib/karafka/pro/processing/partitioner.rb +60 -0
- data/lib/karafka/pro/processing/scheduler.rb +56 -0
- data/lib/karafka/pro/routing/builder_extensions.rb +30 -0
- data/lib/karafka/pro/routing/topic_extensions.rb +38 -0
- data/lib/karafka/pro.rb +13 -0
- data/lib/karafka/process.rb +1 -0
- data/lib/karafka/processing/coordinator.rb +88 -0
- data/lib/karafka/processing/coordinators_buffer.rb +54 -0
- data/lib/karafka/processing/executor.rb +118 -0
- data/lib/karafka/processing/executors_buffer.rb +88 -0
- data/lib/karafka/processing/jobs/base.rb +51 -0
- data/lib/karafka/processing/jobs/consume.rb +42 -0
- data/lib/karafka/processing/jobs/revoked.rb +22 -0
- data/lib/karafka/processing/jobs/shutdown.rb +23 -0
- data/lib/karafka/processing/jobs_builder.rb +29 -0
- data/lib/karafka/processing/jobs_queue.rb +144 -0
- data/lib/karafka/processing/partitioner.rb +22 -0
- data/lib/karafka/processing/result.rb +29 -0
- data/lib/karafka/processing/scheduler.rb +22 -0
- data/lib/karafka/processing/worker.rb +88 -0
- data/lib/karafka/processing/workers_batch.rb +27 -0
- data/lib/karafka/railtie.rb +113 -0
- data/lib/karafka/routing/builder.rb +15 -24
- data/lib/karafka/routing/consumer_group.rb +11 -19
- data/lib/karafka/routing/consumer_mapper.rb +1 -2
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +53 -0
- data/lib/karafka/routing/subscription_groups_builder.rb +53 -0
- data/lib/karafka/routing/topic.rb +61 -24
- data/lib/karafka/routing/topics.rb +38 -0
- data/lib/karafka/runner.rb +51 -0
- data/lib/karafka/serialization/json/deserializer.rb +6 -15
- data/lib/karafka/server.rb +67 -26
- data/lib/karafka/setup/config.rb +147 -175
- data/lib/karafka/status.rb +14 -5
- data/lib/karafka/templates/example_consumer.rb.erb +16 -0
- data/lib/karafka/templates/karafka.rb.erb +15 -51
- data/lib/karafka/time_trackers/base.rb +19 -0
- data/lib/karafka/time_trackers/pause.rb +92 -0
- data/lib/karafka/time_trackers/poll.rb +65 -0
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +38 -17
- data.tar.gz.sig +0 -0
- metadata +118 -120
- metadata.gz.sig +0 -0
- data/MIT-LICENCE +0 -18
- data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
- data/lib/karafka/attributes_map.rb +0 -63
- data/lib/karafka/backends/inline.rb +0 -16
- data/lib/karafka/base_responder.rb +0 -226
- data/lib/karafka/cli/flow.rb +0 -48
- data/lib/karafka/cli/missingno.rb +0 -19
- data/lib/karafka/code_reloader.rb +0 -67
- data/lib/karafka/connection/api_adapter.rb +0 -158
- data/lib/karafka/connection/batch_delegator.rb +0 -55
- data/lib/karafka/connection/builder.rb +0 -23
- data/lib/karafka/connection/message_delegator.rb +0 -36
- data/lib/karafka/consumers/batch_metadata.rb +0 -10
- data/lib/karafka/consumers/callbacks.rb +0 -71
- data/lib/karafka/consumers/includer.rb +0 -64
- data/lib/karafka/consumers/responders.rb +0 -24
- data/lib/karafka/consumers/single_params.rb +0 -15
- data/lib/karafka/contracts/responder_usage.rb +0 -54
- data/lib/karafka/fetcher.rb +0 -42
- data/lib/karafka/helpers/class_matcher.rb +0 -88
- data/lib/karafka/helpers/config_retriever.rb +0 -46
- data/lib/karafka/helpers/inflector.rb +0 -26
- data/lib/karafka/instrumentation/stdout_listener.rb +0 -140
- data/lib/karafka/params/batch_metadata.rb +0 -26
- data/lib/karafka/params/builders/batch_metadata.rb +0 -30
- data/lib/karafka/params/builders/params.rb +0 -38
- data/lib/karafka/params/builders/params_batch.rb +0 -25
- data/lib/karafka/params/params_batch.rb +0 -60
- data/lib/karafka/patches/ruby_kafka.rb +0 -47
- data/lib/karafka/persistence/client.rb +0 -29
- data/lib/karafka/persistence/consumers.rb +0 -45
- data/lib/karafka/persistence/topics.rb +0 -48
- data/lib/karafka/responders/builder.rb +0 -36
- data/lib/karafka/responders/topic.rb +0 -55
- data/lib/karafka/routing/topic_mapper.rb +0 -53
- data/lib/karafka/serialization/json/serializer.rb +0 -31
- data/lib/karafka/setup/configurators/water_drop.rb +0 -36
- data/lib/karafka/templates/application_responder.rb.erb +0 -11
data/bin/benchmarks
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
# Runner for running given benchmark cases
|
|
4
|
+
# Some of the cases require pre-populated data and we populate this in places that need it
|
|
5
|
+
# In other cases we generate this data in a background process, so the partitions data stream
|
|
6
|
+
# is consistent and we don't end up consuming huge batches of a single partition.
|
|
7
|
+
|
|
8
|
+
require 'open3'
|
|
9
|
+
require 'pathname'
|
|
10
|
+
|
|
11
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
|
|
13
|
+
|
|
14
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
|
15
|
+
|
|
16
|
+
# Load all the benchmarks
|
|
17
|
+
benchmarks = Dir[ROOT_PATH.join('spec/benchmarks/**/*.rb')]
|
|
18
|
+
|
|
19
|
+
# If filter is provided, apply
|
|
20
|
+
benchmarks.delete_if { |name| !name.include?(ARGV[0]) } if ARGV[0]
|
|
21
|
+
|
|
22
|
+
raise ArgumentError, "No benchmarks with filter: #{ARGV[0]}" if benchmarks.empty?
|
|
23
|
+
|
|
24
|
+
# We may skip seeding if we are running the benchmarks multiple times, then since we do not
|
|
25
|
+
# commit offsets we can skip generating more data
|
|
26
|
+
if ENV['SEED']
|
|
27
|
+
require 'spec/benchmarks_helper'
|
|
28
|
+
|
|
29
|
+
# We need to setup karafka here to have producer for data seeding
|
|
30
|
+
setup_karafka
|
|
31
|
+
|
|
32
|
+
# This takes some time but needs to run only once per benchmark session
|
|
33
|
+
puts 'Seeding benchmarks data...'
|
|
34
|
+
|
|
35
|
+
producer = Karafka::App.producer
|
|
36
|
+
|
|
37
|
+
# We make our data json compatible so we can also benchmark serialization
|
|
38
|
+
elements = Array.new(100_000) { { a: :b }.to_json }
|
|
39
|
+
|
|
40
|
+
# We do not populate data of benchmarks_0_10 as we use it with life-stream data only
|
|
41
|
+
%w[
|
|
42
|
+
benchmarks_00_01
|
|
43
|
+
benchmarks_00_05
|
|
44
|
+
].each do |topic_name|
|
|
45
|
+
partitions_count = topic_name.split('_').last.to_i
|
|
46
|
+
|
|
47
|
+
partitions_count.times do |partition|
|
|
48
|
+
puts "Seeding #{topic_name}:#{partition}"
|
|
49
|
+
|
|
50
|
+
elements.each_slice(10_000) do |data_slice|
|
|
51
|
+
data = data_slice.map do |data|
|
|
52
|
+
{ topic: topic_name, payload: data, partition: partition }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
producer.buffer_many(data)
|
|
56
|
+
producer.flush_sync
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Selects requested benchmarks and runs them one after another
|
|
63
|
+
benchmarks.each do |benchmark_path|
|
|
64
|
+
puts "Running #{benchmark_path.gsub("#{ROOT_PATH}/spec/benchmarks/", '')}"
|
|
65
|
+
|
|
66
|
+
benchmark = "bundle exec ruby -r ./spec/benchmarks_helper.rb #{benchmark_path}"
|
|
67
|
+
|
|
68
|
+
Open3.popen3(benchmark) do |stdin, stdout, stderr, thread|
|
|
69
|
+
t1 = Thread.new do
|
|
70
|
+
while line = stdout.gets
|
|
71
|
+
puts(line)
|
|
72
|
+
end
|
|
73
|
+
rescue IOError
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
t2 = Thread.new do
|
|
77
|
+
while line = stderr.gets
|
|
78
|
+
puts(line)
|
|
79
|
+
end
|
|
80
|
+
rescue IOError
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
thread.join
|
|
84
|
+
end
|
|
85
|
+
end
|
data/bin/create_token
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'openssl'
|
|
4
|
+
require 'base64'
|
|
5
|
+
require 'json'
|
|
6
|
+
require 'date'
|
|
7
|
+
|
|
8
|
+
PRIVATE_KEY_LOCATION = File.join(Dir.home, '.ssh', 'karafka-pro', 'id_rsa')
|
|
9
|
+
|
|
10
|
+
# Name of the entity that acquires the license
|
|
11
|
+
ENTITY = ARGV[0]
|
|
12
|
+
|
|
13
|
+
raise ArgumentError, 'Entity missing' if ENTITY.nil? || ENTITY.empty?
|
|
14
|
+
|
|
15
|
+
pro_token_data = { entity: ENTITY }
|
|
16
|
+
|
|
17
|
+
# This code uses my private key to generate a new token for Karafka Pro capabilities
|
|
18
|
+
private_key = OpenSSL::PKey::RSA.new(File.read(PRIVATE_KEY_LOCATION))
|
|
19
|
+
|
|
20
|
+
bin_key = private_key.private_encrypt(pro_token_data.to_json)
|
|
21
|
+
|
|
22
|
+
puts Base64.encode64(bin_key)
|
data/bin/integrations
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
# Runner to run integration specs in parallel
|
|
4
|
+
|
|
5
|
+
# Part of integration specs run pristine without bundler.
|
|
6
|
+
# If we would run bundle exec when running this code, bundler would inject its own context
|
|
7
|
+
# into them, messing things up heavily
|
|
8
|
+
raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
|
|
9
|
+
|
|
10
|
+
require 'open3'
|
|
11
|
+
require 'fileutils'
|
|
12
|
+
require 'pathname'
|
|
13
|
+
require 'tmpdir'
|
|
14
|
+
require 'etc'
|
|
15
|
+
|
|
16
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
|
17
|
+
|
|
18
|
+
# How many child processes with integration specs do we want to run in parallel
|
|
19
|
+
# When the value is high, there's a problem with thread allocation on Github CI, tht is why
|
|
20
|
+
# we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
|
|
21
|
+
# of CPU
|
|
22
|
+
CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
|
|
23
|
+
|
|
24
|
+
# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
|
|
25
|
+
MAX_BUFFER_OUTPUT = 51_200
|
|
26
|
+
|
|
27
|
+
# Abstraction around a single test scenario execution process
|
|
28
|
+
class Scenario
|
|
29
|
+
# How long a scenario can run before we kill it
|
|
30
|
+
# This is a fail-safe just in case something would hang
|
|
31
|
+
MAX_RUN_TIME = 3 * 60 # 3 minutes tops
|
|
32
|
+
|
|
33
|
+
# There are rare cases where Karafka may force shutdown for some of the integration cases
|
|
34
|
+
# This includes exactly those
|
|
35
|
+
EXIT_CODES = {
|
|
36
|
+
default: [0],
|
|
37
|
+
'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
|
|
38
|
+
'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
|
|
39
|
+
'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
|
|
40
|
+
'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
|
|
41
|
+
}.freeze
|
|
42
|
+
|
|
43
|
+
private_constant :MAX_RUN_TIME, :EXIT_CODES
|
|
44
|
+
|
|
45
|
+
# Creates scenario instance and runs in the background process
|
|
46
|
+
#
|
|
47
|
+
# @param path [String] path to the scenarios file
|
|
48
|
+
def initialize(path)
|
|
49
|
+
@path = path
|
|
50
|
+
# Last 1024 characters from stdout
|
|
51
|
+
@stdout_tail = ''
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Starts running given scenario in a separate process
|
|
55
|
+
def start
|
|
56
|
+
@stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
|
|
57
|
+
@started_at = current_time
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# @return [String] integration spec name
|
|
61
|
+
def name
|
|
62
|
+
@path.gsub("#{ROOT_PATH}/spec/integrations/", '')
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @return [Boolean] true if spec is pristine
|
|
66
|
+
def pristine?
|
|
67
|
+
scenario_dir = File.dirname(@path)
|
|
68
|
+
|
|
69
|
+
# If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
|
|
70
|
+
# to run bundle install, etc in order to run it
|
|
71
|
+
File.exist?(File.join(scenario_dir, 'Gemfile'))
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# @return [Boolean] did this scenario finished or is it still running
|
|
75
|
+
def finished?
|
|
76
|
+
# If the thread is running too long, kill it
|
|
77
|
+
if current_time - @started_at > MAX_RUN_TIME
|
|
78
|
+
@wait_thr.kill
|
|
79
|
+
|
|
80
|
+
begin
|
|
81
|
+
Process.kill('TERM', pid)
|
|
82
|
+
# It may finish right after we want to kill it, that's why we ignore this
|
|
83
|
+
rescue Errno::ESRCH
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# We read it so it won't grow as we use our default logger that prints to both test.log and
|
|
88
|
+
# to stdout. Otherwise after reaching the buffer size, it would hang
|
|
89
|
+
buffer = ''
|
|
90
|
+
@stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
|
|
91
|
+
@stdout_tail << buffer
|
|
92
|
+
@stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
|
|
93
|
+
|
|
94
|
+
!@wait_thr.alive?
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @return [Boolean] did this scenario finish successfully or not
|
|
98
|
+
def success?
|
|
99
|
+
expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
|
|
100
|
+
|
|
101
|
+
expected_exit_codes.include?(exit_code)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# @return [Integer] pid of the process of this scenario
|
|
105
|
+
def pid
|
|
106
|
+
@wait_thr.pid
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# @return [Integer] exit code of the process running given scenario
|
|
110
|
+
def exit_code
|
|
111
|
+
# There may be no exit status if we killed the thread
|
|
112
|
+
@wait_thr.value&.exitstatus || 123
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Prints a status report when scenario is finished and stdout if it failed
|
|
116
|
+
def report
|
|
117
|
+
if success?
|
|
118
|
+
print "\e[#{32}m#{'.'}\e[0m"
|
|
119
|
+
else
|
|
120
|
+
buffer = ''
|
|
121
|
+
|
|
122
|
+
@stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
|
|
123
|
+
|
|
124
|
+
puts
|
|
125
|
+
puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
|
|
126
|
+
puts "Exit code: #{exit_code}"
|
|
127
|
+
puts @stdout_tail
|
|
128
|
+
puts buffer
|
|
129
|
+
puts
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
private
|
|
134
|
+
|
|
135
|
+
# Sets up a proper environment for a given spec to run and returns the run command
|
|
136
|
+
# @return [String] run command
|
|
137
|
+
def init_and_build_cmd
|
|
138
|
+
# If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
|
|
139
|
+
# to run bundle install, etc in order to run it
|
|
140
|
+
if pristine?
|
|
141
|
+
scenario_dir = File.dirname(@path)
|
|
142
|
+
# We copy the spec into a temp dir, not to pollute the spec location with logs, etc
|
|
143
|
+
temp_dir = Dir.mktmpdir
|
|
144
|
+
file_name = File.basename(@path)
|
|
145
|
+
|
|
146
|
+
FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
|
|
147
|
+
|
|
148
|
+
<<~CMD
|
|
149
|
+
cd #{temp_dir} &&
|
|
150
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
|
151
|
+
BUNDLE_AUTO_INSTALL=true \
|
|
152
|
+
PRISTINE_MODE=true \
|
|
153
|
+
bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{file_name}
|
|
154
|
+
CMD
|
|
155
|
+
else
|
|
156
|
+
<<~CMD
|
|
157
|
+
KARAFKA_GEM_DIR=#{ROOT_PATH} \
|
|
158
|
+
bundle exec ruby -r ./spec/integrations_helper.rb #{@path}
|
|
159
|
+
CMD
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# @return [Float] current machine time
|
|
164
|
+
def current_time
|
|
165
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Load all the specs
|
|
170
|
+
specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
|
|
171
|
+
|
|
172
|
+
# If filters is provided, apply
|
|
173
|
+
# Allows to provide several filters one after another and applies all of them
|
|
174
|
+
ARGV.each do |filter|
|
|
175
|
+
specs.delete_if { |name| !name.include?(filter) }
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
|
179
|
+
|
|
180
|
+
# Randomize order
|
|
181
|
+
seed = (ENV['SEED'] || rand(0..10_000)).to_i
|
|
182
|
+
|
|
183
|
+
puts "Random seed: #{seed}"
|
|
184
|
+
|
|
185
|
+
scenarios = specs
|
|
186
|
+
.shuffle(random: Random.new(seed))
|
|
187
|
+
.map { |integration_test| Scenario.new(integration_test) }
|
|
188
|
+
|
|
189
|
+
regulars = scenarios.reject(&:pristine?)
|
|
190
|
+
pristine = scenarios.select(&:pristine?)
|
|
191
|
+
|
|
192
|
+
active_scenarios = []
|
|
193
|
+
finished_scenarios = []
|
|
194
|
+
|
|
195
|
+
while finished_scenarios.size < scenarios.size
|
|
196
|
+
# If we have space to run another scenario, we add it
|
|
197
|
+
if active_scenarios.size < CONCURRENCY
|
|
198
|
+
scenario = nil
|
|
199
|
+
# We can run only one pristine at the same time due to concurrency issues within bundler
|
|
200
|
+
# Since they usually take longer than others, we try to run them as fast as possible when there
|
|
201
|
+
# is a slot
|
|
202
|
+
scenario = pristine.pop unless active_scenarios.any?(&:pristine?)
|
|
203
|
+
scenario ||= regulars.pop
|
|
204
|
+
|
|
205
|
+
if scenario
|
|
206
|
+
scenario.start
|
|
207
|
+
active_scenarios << scenario
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
active_scenarios.select(&:finished?).each do |exited|
|
|
212
|
+
scenario = active_scenarios.delete(exited)
|
|
213
|
+
scenario.report
|
|
214
|
+
finished_scenarios << scenario
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
sleep(0.1)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
failed_scenarios = finished_scenarios.reject(&:success?)
|
|
221
|
+
|
|
222
|
+
# Report once more on the failed jobs
|
|
223
|
+
# This will only list scenarios that failed without printing their stdout here.
|
|
224
|
+
if failed_scenarios.empty?
|
|
225
|
+
puts
|
|
226
|
+
else
|
|
227
|
+
puts "\nFailed scenarios:\n\n"
|
|
228
|
+
|
|
229
|
+
failed_scenarios.each do |scenario|
|
|
230
|
+
puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
puts
|
|
234
|
+
|
|
235
|
+
# Exit with 1 if not all scenarios were successful
|
|
236
|
+
exit 1
|
|
237
|
+
end
|
data/bin/karafka
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
require 'karafka'
|
|
4
4
|
|
|
5
|
+
# We set this to indicate, that the process in which we are (whatever it does) was started using
|
|
6
|
+
# our bin/karafka cli
|
|
7
|
+
ENV['KARAFKA_CLI'] = 'true'
|
|
8
|
+
|
|
5
9
|
# If there is a boot file, we need to require it as we expect it to contain
|
|
6
10
|
# Karafka app setup, routes, etc
|
|
7
11
|
if File.exist?(Karafka.boot_file)
|
data/bin/scenario
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
# Runner for non-parallel execution of a single scenario.
|
|
4
|
+
# It prints all the info stdout, etc and basically replaces itself with the scenario execution.
|
|
5
|
+
# It is useful when we work with a single spec and we need all the debug info
|
|
6
|
+
|
|
7
|
+
raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
|
|
8
|
+
|
|
9
|
+
require 'open3'
|
|
10
|
+
require 'fileutils'
|
|
11
|
+
require 'pathname'
|
|
12
|
+
require 'tmpdir'
|
|
13
|
+
require 'etc'
|
|
14
|
+
|
|
15
|
+
ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
|
|
16
|
+
|
|
17
|
+
# Load all the specs
|
|
18
|
+
specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
|
|
19
|
+
|
|
20
|
+
# If filters is provided, apply
|
|
21
|
+
# Allows to provide several filters one after another and applies all of them
|
|
22
|
+
ARGV.each do |filter|
|
|
23
|
+
specs.delete_if { |name| !name.include?(filter) }
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
|
|
27
|
+
raise ArgumentError, "Many specs found with filters: #{ARGV.join(', ')}" if specs.size != 1
|
|
28
|
+
|
|
29
|
+
exec("bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{specs[0]}")
|
data/bin/stress_many
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Runs integration specs in an endless loop
|
|
4
|
+
# This allows us to ensure (after long enough time) that the integrations test suit is stable and
|
|
5
|
+
# that there are no anomalies when running it for a long period of time
|
|
6
|
+
|
|
7
|
+
set -e
|
|
8
|
+
|
|
9
|
+
while :
|
|
10
|
+
do
|
|
11
|
+
clear
|
|
12
|
+
bin/integrations $1
|
|
13
|
+
done
|
data/bin/stress_one
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Runs a single integration spec in an endless loop
|
|
4
|
+
# This allows us to ensure (after long enough time) that the integration spec is stable and
|
|
5
|
+
# that there are no anomalies when running it for a long period of time
|
|
6
|
+
|
|
7
|
+
set -e
|
|
8
|
+
|
|
9
|
+
while :
|
|
10
|
+
do
|
|
11
|
+
clear
|
|
12
|
+
bin/scenario $1
|
|
13
|
+
done
|
data/bin/wait_for_kafka
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# This script allows us to wait for Kafka docker to fully be ready
|
|
4
|
+
# We consider it fully ready when all our topics that need to be created are created as expected
|
|
5
|
+
|
|
6
|
+
KAFKA_NAME='karafka_20_kafka'
|
|
7
|
+
ZOOKEEPER='zookeeper:2181'
|
|
8
|
+
LIST_CMD="kafka-topics.sh --list --zookeeper $ZOOKEEPER"
|
|
9
|
+
|
|
10
|
+
# Take the number of topics that we need to create prior to running anything
|
|
11
|
+
TOPICS_COUNT=`cat docker-compose.yml | grep -E -i 'integrations_|benchmarks_' | wc -l`
|
|
12
|
+
|
|
13
|
+
# And wait until all of them are created
|
|
14
|
+
until (((`docker exec $KAFKA_NAME $LIST_CMD | wc -l`) >= $TOPICS_COUNT));
|
|
15
|
+
do
|
|
16
|
+
echo "Waiting for Kafka to create all the needed topics..."
|
|
17
|
+
sleep 1
|
|
18
|
+
done
|
|
19
|
+
|
|
20
|
+
echo "All the needed topics created."
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
-----BEGIN RSA PUBLIC KEY-----
|
|
2
|
+
MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
|
|
3
|
+
tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
|
|
4
|
+
ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
|
|
5
|
+
Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
|
|
6
|
+
8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
|
|
7
|
+
BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
|
|
8
|
+
2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
|
|
9
|
+
2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
|
|
10
|
+
aQtjlQWKPx5xAgMBAAE=
|
|
11
|
+
-----END RSA PUBLIC KEY-----
|
data/config/errors.yml
CHANGED
|
@@ -1,41 +1,56 @@
|
|
|
1
1
|
en:
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
2
|
+
validations:
|
|
3
|
+
config:
|
|
4
|
+
missing: needs to be present
|
|
5
|
+
client_id_format: 'needs to be a string with a Kafka accepted format'
|
|
6
|
+
license.entity_format: needs to be a string
|
|
7
|
+
license.token_format: needs to be either false or a string
|
|
8
|
+
license.expires_on_format: needs to be a valid date
|
|
9
|
+
concurrency_format: needs to be an integer bigger than 0
|
|
10
|
+
consumer_mapper_format: needs to be present
|
|
11
|
+
consumer_persistence_format: needs to be either true or false
|
|
12
|
+
pause_timeout_format: needs to be an integer bigger than 0
|
|
13
|
+
pause_max_timeout_format: needs to be an integer bigger than 0
|
|
14
|
+
pause_with_exponential_backoff_format: needs to be either true or false
|
|
15
|
+
shutdown_timeout_format: needs to be an integer bigger than 0
|
|
16
|
+
max_wait_time_format: needs to be an integer bigger than 0
|
|
17
|
+
kafka_format: needs to be a filled hash
|
|
18
|
+
internal.status_format: needs to be present
|
|
19
|
+
internal.process_format: needs to be present
|
|
20
|
+
internal.routing.builder_format: needs to be present
|
|
21
|
+
internal.routing.subscription_groups_builder_format: needs to be present
|
|
22
|
+
key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
|
|
23
|
+
max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
|
|
24
|
+
shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
|
|
25
|
+
|
|
26
|
+
server_cli_options:
|
|
27
|
+
missing: needs to be present
|
|
28
|
+
consumer_groups_inclusion: Unknown consumer group
|
|
29
|
+
|
|
30
|
+
consumer_group_topic:
|
|
31
|
+
missing: needs to be present
|
|
32
|
+
name_format: 'needs to be a string with a Kafka accepted format'
|
|
33
|
+
deserializer_format: needs to be present
|
|
34
|
+
manual_offset_management_format: needs to be either true or false
|
|
35
|
+
consumer_format: needs to be present
|
|
36
|
+
id_format: 'needs to be a string with a Kafka accepted format'
|
|
37
|
+
initial_offset_format: needs to be either earliest or latest
|
|
38
|
+
|
|
39
|
+
consumer_group:
|
|
40
|
+
missing: needs to be present
|
|
41
|
+
topics_names_not_unique: all topic names within a single consumer group must be unique
|
|
42
|
+
id_format: 'needs to be a string with a Kafka accepted format'
|
|
43
|
+
topics_format: needs to be a non-empty array
|
|
44
|
+
|
|
45
|
+
job_options:
|
|
46
|
+
missing: needs to be present
|
|
47
|
+
dispatch_method_format: needs to be either :produce_async or :produce_sync
|
|
48
|
+
partitioner_format: 'needs to respond to #call'
|
|
49
|
+
partition_key_type_format: 'needs to be either :key or :partition_key'
|
|
50
|
+
|
|
51
|
+
test:
|
|
52
|
+
missing: needs to be present
|
|
53
|
+
id_format: needs to be a String
|
|
54
|
+
|
|
55
|
+
pro_consumer_group_topic:
|
|
56
|
+
consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
|
data/docker-compose.yml
CHANGED
|
@@ -1,17 +1,53 @@
|
|
|
1
1
|
version: '2'
|
|
2
2
|
services:
|
|
3
3
|
zookeeper:
|
|
4
|
+
container_name: karafka_20_zookeeper
|
|
4
5
|
image: wurstmeister/zookeeper
|
|
5
6
|
ports:
|
|
6
|
-
-
|
|
7
|
+
- '2181:2181'
|
|
7
8
|
kafka:
|
|
8
|
-
|
|
9
|
+
container_name: karafka_20_kafka
|
|
10
|
+
image: wurstmeister/kafka
|
|
9
11
|
ports:
|
|
10
|
-
-
|
|
12
|
+
- '9092:9092'
|
|
11
13
|
environment:
|
|
12
14
|
KAFKA_ADVERTISED_HOST_NAME: localhost
|
|
13
15
|
KAFKA_ADVERTISED_PORT: 9092
|
|
14
16
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
|
15
17
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
|
18
|
+
KAFKA_CREATE_TOPICS:
|
|
19
|
+
"integrations_00_02:2:1,\
|
|
20
|
+
integrations_01_02:2:1,\
|
|
21
|
+
integrations_02_02:2:1,\
|
|
22
|
+
integrations_03_02:2:1,\
|
|
23
|
+
integrations_04_02:2:1,\
|
|
24
|
+
integrations_05_02:2:1,\
|
|
25
|
+
integrations_06_02:2:1,\
|
|
26
|
+
integrations_07_02:2:1,\
|
|
27
|
+
integrations_08_02:2:1,\
|
|
28
|
+
integrations_09_02:2:1,\
|
|
29
|
+
integrations_10_02:2:1,\
|
|
30
|
+
integrations_11_02:2:1,\
|
|
31
|
+
integrations_12_02:2:1,\
|
|
32
|
+
integrations_13_02:2:1,\
|
|
33
|
+
integrations_14_02:2:1,\
|
|
34
|
+
integrations_15_02:2:1,\
|
|
35
|
+
integrations_16_02:2:1,\
|
|
36
|
+
integrations_17_02:2:1,\
|
|
37
|
+
integrations_18_02:2:1,\
|
|
38
|
+
integrations_19_02:2:1,\
|
|
39
|
+
integrations_20_02:2:1,\
|
|
40
|
+
integrations_21_02:2:1,\
|
|
41
|
+
integrations_00_03:3:1,\
|
|
42
|
+
integrations_01_03:3:1,\
|
|
43
|
+
integrations_02_03:3:1,\
|
|
44
|
+
integrations_03_03:3:1,\
|
|
45
|
+
integrations_04_03:3:1,\
|
|
46
|
+
integrations_00_10:10:1,\
|
|
47
|
+
integrations_01_10:10:1,\
|
|
48
|
+
benchmarks_00_01:1:1,\
|
|
49
|
+
benchmarks_00_05:5:1,\
|
|
50
|
+
benchmarks_01_05:5:1,\
|
|
51
|
+
benchmarks_00_10:10:1"
|
|
16
52
|
volumes:
|
|
17
53
|
- /var/run/docker.sock:/var/run/docker.sock
|
data/karafka.gemspec
CHANGED
|
@@ -5,29 +5,24 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
5
5
|
|
|
6
6
|
require 'karafka/version'
|
|
7
7
|
|
|
8
|
-
# rubocop:disable Metrics/BlockLength
|
|
9
8
|
Gem::Specification.new do |spec|
|
|
10
9
|
spec.name = 'karafka'
|
|
11
10
|
spec.version = ::Karafka::VERSION
|
|
12
11
|
spec.platform = Gem::Platform::RUBY
|
|
13
|
-
spec.authors = ['Maciej Mensfeld'
|
|
14
|
-
spec.email = %w[maciej@mensfeld.pl
|
|
12
|
+
spec.authors = ['Maciej Mensfeld']
|
|
13
|
+
spec.email = %w[maciej@mensfeld.pl]
|
|
15
14
|
spec.homepage = 'https://karafka.io'
|
|
16
|
-
spec.summary = '
|
|
15
|
+
spec.summary = 'Efficient Kafka processing framework for Ruby and Rails'
|
|
17
16
|
spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
|
|
18
|
-
spec.
|
|
17
|
+
spec.licenses = ['LGPL-3.0', 'Commercial']
|
|
19
18
|
|
|
20
|
-
spec.add_dependency '
|
|
21
|
-
spec.add_dependency '
|
|
22
|
-
spec.add_dependency '
|
|
23
|
-
spec.add_dependency '
|
|
24
|
-
spec.add_dependency '
|
|
25
|
-
spec.add_dependency 'ruby-kafka', '>= 1.3.0'
|
|
26
|
-
spec.add_dependency 'thor', '>= 1.1'
|
|
27
|
-
spec.add_dependency 'waterdrop', '~> 1.4'
|
|
28
|
-
spec.add_dependency 'zeitwerk', '~> 2.4'
|
|
19
|
+
spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
|
|
20
|
+
spec.add_dependency 'rdkafka', '>= 0.12'
|
|
21
|
+
spec.add_dependency 'thor', '>= 0.20'
|
|
22
|
+
spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
|
|
23
|
+
spec.add_dependency 'zeitwerk', '~> 2.3'
|
|
29
24
|
|
|
30
|
-
spec.required_ruby_version = '>= 2.7'
|
|
25
|
+
spec.required_ruby_version = '>= 2.7.0'
|
|
31
26
|
|
|
32
27
|
if $PROGRAM_NAME.end_with?('gem')
|
|
33
28
|
spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
|
|
@@ -35,7 +30,7 @@ Gem::Specification.new do |spec|
|
|
|
35
30
|
|
|
36
31
|
spec.cert_chain = %w[certs/mensfeld.pem]
|
|
37
32
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
|
|
38
|
-
spec.executables =
|
|
33
|
+
spec.executables = %w[karafka]
|
|
39
34
|
spec.require_paths = %w[lib]
|
|
40
35
|
|
|
41
36
|
spec.metadata = {
|
|
@@ -43,4 +38,3 @@ Gem::Specification.new do |spec|
|
|
|
43
38
|
'rubygems_mfa_required' => 'true'
|
|
44
39
|
}
|
|
45
40
|
end
|
|
46
|
-
# rubocop:enable Metrics/BlockLength
|