karafka 2.0.23 → 2.0.24
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +2 -0
- data/CHANGELOG.md +32 -1
- data/Gemfile.lock +8 -7
- data/README.md +3 -3
- data/config/{errors.yml → locales/errors.yml} +1 -1
- data/config/locales/pro_errors.yml +18 -0
- data/docker-compose.yml +3 -0
- data/karafka.gemspec +2 -2
- data/lib/karafka/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/admin.rb +2 -4
- data/lib/karafka/app.rb +15 -4
- data/lib/karafka/base_consumer.rb +36 -0
- data/lib/karafka/connection/listener.rb +1 -1
- data/lib/karafka/contracts/config.rb +1 -1
- data/lib/karafka/contracts/consumer_group.rb +1 -1
- data/lib/karafka/contracts/server_cli_options.rb +1 -1
- data/lib/karafka/contracts/topic.rb +1 -1
- data/lib/karafka/instrumentation/logger_listener.rb +32 -0
- data/lib/karafka/instrumentation/notifications.rb +3 -0
- data/lib/karafka/messages/message.rb +14 -2
- data/lib/karafka/messages/parser.rb +14 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/pro/encryption/cipher.rb +58 -0
- data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
- data/lib/karafka/pro/encryption/errors.rb +24 -0
- data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
- data/lib/karafka/pro/encryption/setup/config.rb +48 -0
- data/lib/karafka/pro/encryption.rb +47 -0
- data/lib/karafka/pro/loader.rb +22 -1
- data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/default.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/lrj.rb +1 -1
- data/lib/karafka/pro/processing/strategies/lrj_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/mom.rb +1 -1
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contract.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +2 -2
- data/lib/karafka/processing/executor.rb +1 -1
- data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/default.rb +1 -1
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/strategies/dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/mom.rb +1 -1
- data/lib/karafka/processing/worker.rb +1 -1
- data/lib/karafka/railtie.rb +3 -0
- data/lib/karafka/routing/builder.rb +1 -1
- data/lib/karafka/routing/consumer_group.rb +3 -3
- data/lib/karafka/routing/features/active_job/contract.rb +1 -1
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +1 -1
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +1 -1
- data/lib/karafka/server.rb +14 -14
- data/lib/karafka/setup/config.rb +15 -2
- data/lib/karafka/status.rb +27 -9
- data/lib/karafka/templates/karafka.rb.erb +1 -2
- data/lib/karafka/time_trackers/pause.rb +3 -1
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +16 -7
- metadata.gz.sig +0 -0
data/lib/karafka/status.rb
CHANGED
@@ -10,32 +10,50 @@ module Karafka
|
|
10
10
|
running: :run!,
|
11
11
|
quieting: :quiet!,
|
12
12
|
stopping: :stop!,
|
13
|
-
stopped: :stopped
|
13
|
+
stopped: :stopped!,
|
14
|
+
terminated: :terminate!
|
14
15
|
}.freeze
|
15
16
|
|
16
|
-
|
17
|
+
# Mutex to ensure that state transitions are thread-safe
|
18
|
+
MUTEX = Mutex.new
|
19
|
+
|
20
|
+
private_constant :MUTEX
|
17
21
|
|
18
22
|
# By default we are in the initializing state
|
19
23
|
def initialize
|
20
24
|
initialize!
|
21
25
|
end
|
22
26
|
|
27
|
+
# @return [String] stringified current app status
|
28
|
+
def to_s
|
29
|
+
@status.to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
# Resets the status state
|
33
|
+
# This is used mostly in the integration suite
|
34
|
+
def reset!
|
35
|
+
@status = :initializing
|
36
|
+
end
|
37
|
+
|
23
38
|
STATES.each do |state, transition|
|
24
39
|
define_method :"#{state}?" do
|
25
40
|
@status == state
|
26
41
|
end
|
27
42
|
|
28
43
|
define_method transition do
|
29
|
-
|
30
|
-
|
44
|
+
MUTEX.synchronize do
|
45
|
+
# Do not allow reverse state transitions (we always go one way) or transition to the same
|
46
|
+
# state as currently
|
47
|
+
return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
|
31
48
|
|
32
|
-
|
49
|
+
@status = state
|
33
50
|
|
34
|
-
|
35
|
-
|
36
|
-
|
51
|
+
# Skip on creation (initializing)
|
52
|
+
# We skip as during this state we do not have yet a monitor
|
53
|
+
return if initializing?
|
37
54
|
|
38
|
-
|
55
|
+
Karafka.monitor.instrument("app.#{state}")
|
56
|
+
end
|
39
57
|
end
|
40
58
|
end
|
41
59
|
end
|
@@ -5,8 +5,7 @@
|
|
5
5
|
# If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
|
6
6
|
# command again or refer to the install templates available in the source codes
|
7
7
|
|
8
|
-
ENV['
|
9
|
-
ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
|
8
|
+
ENV['KARAFKA_ENV'] ||= 'development'
|
10
9
|
Bundler.require(:default, ENV['KARAFKA_ENV'])
|
11
10
|
|
12
11
|
# Zeitwerk custom loader for loading the app components before the whole
|
@@ -10,7 +10,7 @@ module Karafka
|
|
10
10
|
# code here, as this is not a frequently used tracker. It is active only once per batch in
|
11
11
|
# case of long-running-jobs and upon errors.
|
12
12
|
class Pause < Base
|
13
|
-
attr_reader :attempt
|
13
|
+
attr_reader :attempt, :current_timeout
|
14
14
|
|
15
15
|
# @param timeout [Integer] how long should we wait when anything went wrong (in ms)
|
16
16
|
# @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
|
@@ -43,6 +43,7 @@ module Karafka
|
|
43
43
|
@started_at = nil
|
44
44
|
@attempt = 0
|
45
45
|
@timeout = timeout
|
46
|
+
@current_timeout = timeout
|
46
47
|
@max_timeout = max_timeout
|
47
48
|
@exponential_backoff = exponential_backoff
|
48
49
|
@mutex = Mutex.new
|
@@ -56,6 +57,7 @@ module Karafka
|
|
56
57
|
# period of time, outside of any regular pausing logic
|
57
58
|
def pause(timeout = backoff_interval)
|
58
59
|
@mutex.synchronize do
|
60
|
+
@current_timeout = timeout
|
59
61
|
@started_at = now
|
60
62
|
@ends_at = @started_at + timeout
|
61
63
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2022-12-
|
38
|
+
date: 2022-12-19 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 2.0.
|
46
|
+
version: 2.0.7
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: 3.0.0
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
requirements:
|
54
54
|
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 2.0.
|
56
|
+
version: 2.0.7
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: 3.0.0
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 2.4.
|
80
|
+
version: 2.4.7
|
81
81
|
- - "<"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: 3.0.0
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 2.4.
|
90
|
+
version: 2.4.7
|
91
91
|
- - "<"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: 3.0.0
|
@@ -147,7 +147,8 @@ files:
|
|
147
147
|
- bin/stress_one
|
148
148
|
- certs/cert_chain.pem
|
149
149
|
- certs/karafka-pro.pem
|
150
|
-
- config/errors.yml
|
150
|
+
- config/locales/errors.yml
|
151
|
+
- config/locales/pro_errors.yml
|
151
152
|
- docker-compose.yml
|
152
153
|
- karafka.gemspec
|
153
154
|
- lib/active_job/karafka.rb
|
@@ -204,6 +205,7 @@ files:
|
|
204
205
|
- lib/karafka/messages/message.rb
|
205
206
|
- lib/karafka/messages/messages.rb
|
206
207
|
- lib/karafka/messages/metadata.rb
|
208
|
+
- lib/karafka/messages/parser.rb
|
207
209
|
- lib/karafka/messages/seek.rb
|
208
210
|
- lib/karafka/patches/rdkafka/bindings.rb
|
209
211
|
- lib/karafka/patches/rdkafka/consumer.rb
|
@@ -212,6 +214,13 @@ files:
|
|
212
214
|
- lib/karafka/pro/active_job/dispatcher.rb
|
213
215
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
214
216
|
- lib/karafka/pro/base_consumer.rb
|
217
|
+
- lib/karafka/pro/encryption.rb
|
218
|
+
- lib/karafka/pro/encryption/cipher.rb
|
219
|
+
- lib/karafka/pro/encryption/contracts/config.rb
|
220
|
+
- lib/karafka/pro/encryption/errors.rb
|
221
|
+
- lib/karafka/pro/encryption/messages/middleware.rb
|
222
|
+
- lib/karafka/pro/encryption/messages/parser.rb
|
223
|
+
- lib/karafka/pro/encryption/setup/config.rb
|
215
224
|
- lib/karafka/pro/loader.rb
|
216
225
|
- lib/karafka/pro/performance_tracker.rb
|
217
226
|
- lib/karafka/pro/processing/coordinator.rb
|
metadata.gz.sig
CHANGED
Binary file
|