karafka 2.0.23 → 2.0.26
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +24 -3
- data/.ruby-version +1 -1
- data/CHANGELOG.md +52 -1
- data/Gemfile.lock +14 -12
- data/README.md +6 -4
- data/bin/integrations +8 -0
- data/bin/verify_license_integrity +35 -0
- data/config/{errors.yml → locales/errors.yml} +2 -1
- data/config/locales/pro_errors.yml +18 -0
- data/docker-compose.yml +3 -0
- data/karafka.gemspec +3 -3
- data/lib/karafka/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/admin.rb +16 -14
- data/lib/karafka/app.rb +16 -4
- data/lib/karafka/base_consumer.rb +37 -7
- data/lib/karafka/connection/client.rb +21 -0
- data/lib/karafka/connection/consumer_group_coordinator.rb +7 -1
- data/lib/karafka/connection/listener.rb +5 -4
- data/lib/karafka/connection/listeners_batch.rb +6 -0
- data/lib/karafka/contracts/config.rb +1 -1
- data/lib/karafka/contracts/consumer_group.rb +1 -1
- data/lib/karafka/contracts/server_cli_options.rb +2 -1
- data/lib/karafka/contracts/topic.rb +13 -2
- data/lib/karafka/instrumentation/logger_listener.rb +50 -2
- data/lib/karafka/instrumentation/notifications.rb +17 -7
- data/lib/karafka/instrumentation/proctitle_listener.rb +7 -16
- data/lib/karafka/instrumentation/vendors/datadog/listener.rb +2 -2
- data/lib/karafka/messages/message.rb +14 -2
- data/lib/karafka/messages/parser.rb +14 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/pro/encryption/cipher.rb +58 -0
- data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
- data/lib/karafka/pro/encryption/errors.rb +24 -0
- data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
- data/lib/karafka/pro/encryption/setup/config.rb +48 -0
- data/lib/karafka/pro/encryption.rb +47 -0
- data/lib/karafka/pro/loader.rb +22 -1
- data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +6 -1
- data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/default.rb +7 -1
- data/lib/karafka/pro/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/lrj.rb +6 -1
- data/lib/karafka/pro/processing/strategies/lrj_mom.rb +6 -1
- data/lib/karafka/pro/processing/strategies/mom.rb +1 -1
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contract.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +2 -2
- data/lib/karafka/process.rb +3 -1
- data/lib/karafka/processing/executor.rb +1 -1
- data/lib/karafka/processing/jobs_queue.rb +2 -2
- data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/base.rb +5 -0
- data/lib/karafka/processing/strategies/default.rb +15 -1
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/strategies/dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/mom.rb +1 -1
- data/lib/karafka/processing/worker.rb +3 -1
- data/lib/karafka/railtie.rb +3 -0
- data/lib/karafka/routing/builder.rb +1 -1
- data/lib/karafka/routing/consumer_group.rb +3 -3
- data/lib/karafka/routing/consumer_mapper.rb +0 -10
- data/lib/karafka/routing/features/active_job/contract.rb +1 -1
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +1 -1
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +1 -1
- data/lib/karafka/routing/router.rb +12 -2
- data/lib/karafka/routing/subscription_group.rb +18 -1
- data/lib/karafka/routing/topic.rb +11 -0
- data/lib/karafka/runner.rb +1 -0
- data/lib/karafka/server.rb +27 -18
- data/lib/karafka/setup/config.rb +15 -2
- data/lib/karafka/status.rb +33 -9
- data/lib/karafka/templates/karafka.rb.erb +1 -2
- data/lib/karafka/time_trackers/base.rb +1 -6
- data/lib/karafka/time_trackers/pause.rb +5 -3
- data/lib/karafka/time_trackers/poll.rb +2 -2
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +2 -0
- data.tar.gz.sig +0 -0
- metadata +18 -8
- metadata.gz.sig +0 -0
data/lib/karafka/runner.rb
CHANGED
@@ -19,6 +19,7 @@ module Karafka
|
|
19
19
|
# We aggregate threads here for a supervised shutdown process
|
20
20
|
Karafka::Server.workers = workers
|
21
21
|
Karafka::Server.listeners = listeners
|
22
|
+
Karafka::Server.jobs_queue = jobs_queue
|
22
23
|
|
23
24
|
# All the listener threads need to finish
|
24
25
|
listeners.each(&:join)
|
data/lib/karafka/server.rb
CHANGED
@@ -20,6 +20,9 @@ module Karafka
|
|
20
20
|
# Set of workers
|
21
21
|
attr_accessor :workers
|
22
22
|
|
23
|
+
# Jobs queue
|
24
|
+
attr_accessor :jobs_queue
|
25
|
+
|
23
26
|
# Method which runs app
|
24
27
|
def run
|
25
28
|
self.listeners = []
|
@@ -45,9 +48,9 @@ module Karafka
|
|
45
48
|
|
46
49
|
# We always need to wait for Karafka to stop here since we should wait for the stop running
|
47
50
|
# in a separate thread (or trap context) to indicate everything is closed
|
48
|
-
# Since `#start` is blocking, we
|
51
|
+
# Since `#start` is blocking, we will get here only after the runner is done. This will
|
49
52
|
# not add any performance degradation because of that.
|
50
|
-
|
53
|
+
sleep(0.1) until Karafka::App.terminated?
|
51
54
|
# Try its best to shutdown underlying components before re-raising
|
52
55
|
# rubocop:disable Lint/RescueException
|
53
56
|
rescue Exception => e
|
@@ -75,6 +78,7 @@ module Karafka
|
|
75
78
|
# Initialize the stopping process only if Karafka was running
|
76
79
|
return if Karafka::App.stopping?
|
77
80
|
return if Karafka::App.stopped?
|
81
|
+
return if Karafka::App.terminated?
|
78
82
|
|
79
83
|
Karafka::App.stop!
|
80
84
|
|
@@ -84,13 +88,7 @@ module Karafka
|
|
84
88
|
# their work and if so, we can just return and normal shutdown process will take place
|
85
89
|
# We divide it by 1000 because we use time in ms.
|
86
90
|
((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
|
87
|
-
if listeners.count(&:alive?).zero? &&
|
88
|
-
workers.count(&:alive?).zero?
|
89
|
-
|
90
|
-
Karafka::App.producer.close
|
91
|
-
|
92
|
-
return
|
93
|
-
end
|
91
|
+
return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
|
94
92
|
|
95
93
|
sleep SUPERVISION_SLEEP
|
96
94
|
end
|
@@ -111,8 +109,6 @@ module Karafka
|
|
111
109
|
# This can cause memory leaks and crashes.
|
112
110
|
listeners.each(&:shutdown)
|
113
111
|
|
114
|
-
Karafka::App.producer.close
|
115
|
-
|
116
112
|
# We also do not forcefully terminate everything when running in the embedded mode,
|
117
113
|
# otherwise we would overwrite the shutdown process of the process that started Karafka
|
118
114
|
return unless process.supervised?
|
@@ -122,19 +118,32 @@ module Karafka
|
|
122
118
|
ensure
|
123
119
|
# We need to check if it wasn't an early exit to make sure that only on stop invocation
|
124
120
|
# can change the status after everything is closed
|
125
|
-
|
121
|
+
if timeout
|
122
|
+
Karafka::App.stopped!
|
123
|
+
|
124
|
+
# We close producer as the last thing as it can be used in the notification pipeline
|
125
|
+
# to dispatch state changes, etc
|
126
|
+
Karafka::App.producer.close
|
127
|
+
|
128
|
+
Karafka::App.terminate!
|
129
|
+
end
|
126
130
|
end
|
127
131
|
|
128
132
|
# Quiets the Karafka server.
|
129
|
-
#
|
133
|
+
#
|
134
|
+
# Karafka will stop processing but won't quit the consumer group, so no rebalance will be
|
130
135
|
# triggered until final shutdown.
|
131
136
|
def quiet
|
132
|
-
#
|
133
|
-
|
134
|
-
return if Karafka::App.stopping?
|
135
|
-
return if Karafka::App.stopped?
|
136
|
-
|
137
|
+
# We don't have to safe-guard it with check states as the state transitions work only
|
138
|
+
# in one direction
|
137
139
|
Karafka::App.quiet!
|
140
|
+
|
141
|
+
# We need one more thread to monitor the process and move to quieted once everything
|
142
|
+
# is quiet and no processing is happening anymore
|
143
|
+
Thread.new do
|
144
|
+
sleep(0.1) until listeners.coordinators.all?(&:finished?)
|
145
|
+
Karafka::App.quieted!
|
146
|
+
end
|
138
147
|
end
|
139
148
|
|
140
149
|
private
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -16,7 +16,10 @@ module Karafka
|
|
16
16
|
|
17
17
|
# Defaults for kafka settings, that will be overwritten only if not present already
|
18
18
|
KAFKA_DEFAULTS = {
|
19
|
-
'client.id': 'karafka'
|
19
|
+
'client.id': 'karafka',
|
20
|
+
# We emit the statistics by default, so all the instrumentation and web-ui work out of
|
21
|
+
# the box, without requiring users to take any extra actions aside from enabling.
|
22
|
+
'statistics.interval.ms': 5_000
|
20
23
|
}.freeze
|
21
24
|
|
22
25
|
# Contains settings that should not be used in production but make life easier in dev
|
@@ -130,6 +133,12 @@ module Karafka
|
|
130
133
|
setting :strategy_selector, default: Processing::StrategySelector.new
|
131
134
|
end
|
132
135
|
|
136
|
+
# Things related to operating on messages
|
137
|
+
setting :messages do
|
138
|
+
# Parser is used to convert raw payload prior to deserialization
|
139
|
+
setting :parser, default: Messages::Parser.new
|
140
|
+
end
|
141
|
+
|
133
142
|
# Karafka components for ActiveJob
|
134
143
|
setting :active_job do
|
135
144
|
# option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
|
@@ -155,7 +164,7 @@ module Karafka
|
|
155
164
|
# Will configure all the pro components
|
156
165
|
# This needs to happen before end user configuration as the end user may overwrite some
|
157
166
|
# of the pro defaults with custom components
|
158
|
-
Pro::Loader.
|
167
|
+
Pro::Loader.pre_setup(config) if Karafka.pro?
|
159
168
|
|
160
169
|
configure(&block)
|
161
170
|
merge_kafka_defaults!(config)
|
@@ -164,6 +173,10 @@ module Karafka
|
|
164
173
|
|
165
174
|
configure_components
|
166
175
|
|
176
|
+
# Runs things that need to be executed after config is defined and all the components
|
177
|
+
# are also configured
|
178
|
+
Pro::Loader.post_setup(config) if Karafka.pro?
|
179
|
+
|
167
180
|
Karafka::App.initialized!
|
168
181
|
end
|
169
182
|
|
data/lib/karafka/status.rb
CHANGED
@@ -8,34 +8,58 @@ module Karafka
|
|
8
8
|
initializing: :initialize!,
|
9
9
|
initialized: :initialized!,
|
10
10
|
running: :run!,
|
11
|
+
# will no longer pickup any work, but current work will be finished
|
11
12
|
quieting: :quiet!,
|
13
|
+
# no work is happening but we keep process with the assignments running
|
14
|
+
quiet: :quieted!,
|
15
|
+
# shutdown started
|
12
16
|
stopping: :stop!,
|
13
|
-
|
17
|
+
# all things are done and most of the things except critical are closed
|
18
|
+
stopped: :stopped!,
|
19
|
+
# immediately after this process exists
|
20
|
+
terminated: :terminate!
|
14
21
|
}.freeze
|
15
22
|
|
16
|
-
|
23
|
+
# Mutex to ensure that state transitions are thread-safe
|
24
|
+
MUTEX = Mutex.new
|
25
|
+
|
26
|
+
private_constant :MUTEX
|
17
27
|
|
18
28
|
# By default we are in the initializing state
|
19
29
|
def initialize
|
20
30
|
initialize!
|
21
31
|
end
|
22
32
|
|
33
|
+
# @return [String] stringified current app status
|
34
|
+
def to_s
|
35
|
+
@status.to_s
|
36
|
+
end
|
37
|
+
|
38
|
+
# Resets the status state
|
39
|
+
# This is used mostly in the integration suite
|
40
|
+
def reset!
|
41
|
+
@status = :initializing
|
42
|
+
end
|
43
|
+
|
23
44
|
STATES.each do |state, transition|
|
24
45
|
define_method :"#{state}?" do
|
25
46
|
@status == state
|
26
47
|
end
|
27
48
|
|
28
49
|
define_method transition do
|
29
|
-
|
30
|
-
|
50
|
+
MUTEX.synchronize do
|
51
|
+
# Do not allow reverse state transitions (we always go one way) or transition to the same
|
52
|
+
# state as currently
|
53
|
+
return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
|
31
54
|
|
32
|
-
|
55
|
+
@status = state
|
33
56
|
|
34
|
-
|
35
|
-
|
36
|
-
|
57
|
+
# Skip on creation (initializing)
|
58
|
+
# We skip as during this state we do not have yet a monitor
|
59
|
+
return if initializing?
|
37
60
|
|
38
|
-
|
61
|
+
Karafka.monitor.instrument("app.#{state}")
|
62
|
+
end
|
39
63
|
end
|
40
64
|
end
|
41
65
|
end
|
@@ -5,8 +5,7 @@
|
|
5
5
|
# If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
|
6
6
|
# command again or refer to the install templates available in the source codes
|
7
7
|
|
8
|
-
ENV['
|
9
|
-
ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
|
8
|
+
ENV['KARAFKA_ENV'] ||= 'development'
|
10
9
|
Bundler.require(:default, ENV['KARAFKA_ENV'])
|
11
10
|
|
12
11
|
# Zeitwerk custom loader for loading the app components before the whole
|
@@ -8,12 +8,7 @@ module Karafka
|
|
8
8
|
module TimeTrackers
|
9
9
|
# Base class for all the time-trackers.
|
10
10
|
class Base
|
11
|
-
|
12
|
-
|
13
|
-
# @return [Float] current time in milliseconds
|
14
|
-
def now
|
15
|
-
::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000
|
16
|
-
end
|
11
|
+
include ::Karafka::Core::Helpers::Time
|
17
12
|
end
|
18
13
|
end
|
19
14
|
end
|
@@ -10,7 +10,7 @@ module Karafka
|
|
10
10
|
# code here, as this is not a frequently used tracker. It is active only once per batch in
|
11
11
|
# case of long-running-jobs and upon errors.
|
12
12
|
class Pause < Base
|
13
|
-
attr_reader :attempt
|
13
|
+
attr_reader :attempt, :current_timeout
|
14
14
|
|
15
15
|
# @param timeout [Integer] how long should we wait when anything went wrong (in ms)
|
16
16
|
# @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
|
@@ -43,6 +43,7 @@ module Karafka
|
|
43
43
|
@started_at = nil
|
44
44
|
@attempt = 0
|
45
45
|
@timeout = timeout
|
46
|
+
@current_timeout = timeout
|
46
47
|
@max_timeout = max_timeout
|
47
48
|
@exponential_backoff = exponential_backoff
|
48
49
|
@mutex = Mutex.new
|
@@ -56,7 +57,8 @@ module Karafka
|
|
56
57
|
# period of time, outside of any regular pausing logic
|
57
58
|
def pause(timeout = backoff_interval)
|
58
59
|
@mutex.synchronize do
|
59
|
-
@
|
60
|
+
@current_timeout = timeout
|
61
|
+
@started_at = monotonic_now
|
60
62
|
@ends_at = @started_at + timeout
|
61
63
|
end
|
62
64
|
end
|
@@ -93,7 +95,7 @@ module Karafka
|
|
93
95
|
# @return [Boolean] did the pause expire
|
94
96
|
def expired?
|
95
97
|
@mutex.synchronize do
|
96
|
-
@ends_at ?
|
98
|
+
@ends_at ? monotonic_now >= @ends_at : true
|
97
99
|
end
|
98
100
|
end
|
99
101
|
|
@@ -34,12 +34,12 @@ module Karafka
|
|
34
34
|
# Starts time tracking.
|
35
35
|
def start
|
36
36
|
@attempts += 1
|
37
|
-
@started_at =
|
37
|
+
@started_at = monotonic_now
|
38
38
|
end
|
39
39
|
|
40
40
|
# Stops time tracking of a given piece of code and updates the remaining time.
|
41
41
|
def checkpoint
|
42
|
-
@remaining -= (
|
42
|
+
@remaining -= (monotonic_now - @started_at)
|
43
43
|
end
|
44
44
|
|
45
45
|
# @return [Boolean] If anything went wrong, can we retry after a backoff period or not
|
data/lib/karafka/version.rb
CHANGED
data/lib/karafka.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.26
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date:
|
38
|
+
date: 2023-01-10 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 2.0.
|
46
|
+
version: 2.0.8
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: 3.0.0
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
requirements:
|
54
54
|
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 2.0.
|
56
|
+
version: 2.0.8
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: 3.0.0
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 2.4.
|
80
|
+
version: 2.4.7
|
81
81
|
- - "<"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: 3.0.0
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 2.4.
|
90
|
+
version: 2.4.7
|
91
91
|
- - "<"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: 3.0.0
|
@@ -145,9 +145,11 @@ files:
|
|
145
145
|
- bin/scenario
|
146
146
|
- bin/stress_many
|
147
147
|
- bin/stress_one
|
148
|
+
- bin/verify_license_integrity
|
148
149
|
- certs/cert_chain.pem
|
149
150
|
- certs/karafka-pro.pem
|
150
|
-
- config/errors.yml
|
151
|
+
- config/locales/errors.yml
|
152
|
+
- config/locales/pro_errors.yml
|
151
153
|
- docker-compose.yml
|
152
154
|
- karafka.gemspec
|
153
155
|
- lib/active_job/karafka.rb
|
@@ -204,6 +206,7 @@ files:
|
|
204
206
|
- lib/karafka/messages/message.rb
|
205
207
|
- lib/karafka/messages/messages.rb
|
206
208
|
- lib/karafka/messages/metadata.rb
|
209
|
+
- lib/karafka/messages/parser.rb
|
207
210
|
- lib/karafka/messages/seek.rb
|
208
211
|
- lib/karafka/patches/rdkafka/bindings.rb
|
209
212
|
- lib/karafka/patches/rdkafka/consumer.rb
|
@@ -212,6 +215,13 @@ files:
|
|
212
215
|
- lib/karafka/pro/active_job/dispatcher.rb
|
213
216
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
214
217
|
- lib/karafka/pro/base_consumer.rb
|
218
|
+
- lib/karafka/pro/encryption.rb
|
219
|
+
- lib/karafka/pro/encryption/cipher.rb
|
220
|
+
- lib/karafka/pro/encryption/contracts/config.rb
|
221
|
+
- lib/karafka/pro/encryption/errors.rb
|
222
|
+
- lib/karafka/pro/encryption/messages/middleware.rb
|
223
|
+
- lib/karafka/pro/encryption/messages/parser.rb
|
224
|
+
- lib/karafka/pro/encryption/setup/config.rb
|
215
225
|
- lib/karafka/pro/loader.rb
|
216
226
|
- lib/karafka/pro/performance_tracker.rb
|
217
227
|
- lib/karafka/pro/processing/coordinator.rb
|
@@ -340,7 +350,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
340
350
|
- !ruby/object:Gem::Version
|
341
351
|
version: '0'
|
342
352
|
requirements: []
|
343
|
-
rubygems_version: 3.
|
353
|
+
rubygems_version: 3.4.1
|
344
354
|
signing_key:
|
345
355
|
specification_version: 4
|
346
356
|
summary: Karafka is Ruby and Rails efficient Kafka processing framework.
|
metadata.gz.sig
CHANGED
Binary file
|