karafka 2.0.23 → 2.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +2 -0
- data/CHANGELOG.md +32 -1
- data/Gemfile.lock +8 -7
- data/README.md +3 -3
- data/config/{errors.yml → locales/errors.yml} +1 -1
- data/config/locales/pro_errors.yml +18 -0
- data/docker-compose.yml +3 -0
- data/karafka.gemspec +2 -2
- data/lib/karafka/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/admin.rb +2 -4
- data/lib/karafka/app.rb +15 -4
- data/lib/karafka/base_consumer.rb +36 -0
- data/lib/karafka/connection/listener.rb +1 -1
- data/lib/karafka/contracts/config.rb +1 -1
- data/lib/karafka/contracts/consumer_group.rb +1 -1
- data/lib/karafka/contracts/server_cli_options.rb +1 -1
- data/lib/karafka/contracts/topic.rb +1 -1
- data/lib/karafka/instrumentation/logger_listener.rb +32 -0
- data/lib/karafka/instrumentation/notifications.rb +3 -0
- data/lib/karafka/messages/message.rb +14 -2
- data/lib/karafka/messages/parser.rb +14 -0
- data/lib/karafka/pro/active_job/job_options_contract.rb +1 -1
- data/lib/karafka/pro/encryption/cipher.rb +58 -0
- data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
- data/lib/karafka/pro/encryption/errors.rb +24 -0
- data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
- data/lib/karafka/pro/encryption/setup/config.rb +48 -0
- data/lib/karafka/pro/encryption.rb +47 -0
- data/lib/karafka/pro/loader.rb +22 -1
- data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +1 -1
- data/lib/karafka/pro/processing/strategies/default.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/dlq_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/lrj.rb +1 -1
- data/lib/karafka/pro/processing/strategies/lrj_mom.rb +1 -1
- data/lib/karafka/pro/processing/strategies/mom.rb +1 -1
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contract.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +2 -2
- data/lib/karafka/processing/executor.rb +1 -1
- data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/default.rb +1 -1
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/strategies/dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/mom.rb +1 -1
- data/lib/karafka/processing/worker.rb +1 -1
- data/lib/karafka/railtie.rb +3 -0
- data/lib/karafka/routing/builder.rb +1 -1
- data/lib/karafka/routing/consumer_group.rb +3 -3
- data/lib/karafka/routing/features/active_job/contract.rb +1 -1
- data/lib/karafka/routing/features/dead_letter_queue/contract.rb +1 -1
- data/lib/karafka/routing/features/manual_offset_management/contract.rb +1 -1
- data/lib/karafka/server.rb +14 -14
- data/lib/karafka/setup/config.rb +15 -2
- data/lib/karafka/status.rb +27 -9
- data/lib/karafka/templates/karafka.rb.erb +1 -2
- data/lib/karafka/time_trackers/pause.rb +3 -1
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +16 -7
- metadata.gz.sig +0 -0
data/lib/karafka/status.rb
CHANGED
@@ -10,32 +10,50 @@ module Karafka
|
|
10
10
|
running: :run!,
|
11
11
|
quieting: :quiet!,
|
12
12
|
stopping: :stop!,
|
13
|
-
stopped: :stopped
|
13
|
+
stopped: :stopped!,
|
14
|
+
terminated: :terminate!
|
14
15
|
}.freeze
|
15
16
|
|
16
|
-
|
17
|
+
# Mutex to ensure that state transitions are thread-safe
|
18
|
+
MUTEX = Mutex.new
|
19
|
+
|
20
|
+
private_constant :MUTEX
|
17
21
|
|
18
22
|
# By default we are in the initializing state
|
19
23
|
def initialize
|
20
24
|
initialize!
|
21
25
|
end
|
22
26
|
|
27
|
+
# @return [String] stringified current app status
|
28
|
+
def to_s
|
29
|
+
@status.to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
# Resets the status state
|
33
|
+
# This is used mostly in the integration suite
|
34
|
+
def reset!
|
35
|
+
@status = :initializing
|
36
|
+
end
|
37
|
+
|
23
38
|
STATES.each do |state, transition|
|
24
39
|
define_method :"#{state}?" do
|
25
40
|
@status == state
|
26
41
|
end
|
27
42
|
|
28
43
|
define_method transition do
|
29
|
-
|
30
|
-
|
44
|
+
MUTEX.synchronize do
|
45
|
+
# Do not allow reverse state transitions (we always go one way) or transition to the same
|
46
|
+
# state as currently
|
47
|
+
return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
|
31
48
|
|
32
|
-
|
49
|
+
@status = state
|
33
50
|
|
34
|
-
|
35
|
-
|
36
|
-
|
51
|
+
# Skip on creation (initializing)
|
52
|
+
# We skip as during this state we do not have yet a monitor
|
53
|
+
return if initializing?
|
37
54
|
|
38
|
-
|
55
|
+
Karafka.monitor.instrument("app.#{state}")
|
56
|
+
end
|
39
57
|
end
|
40
58
|
end
|
41
59
|
end
|
@@ -5,8 +5,7 @@
|
|
5
5
|
# If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
|
6
6
|
# command again or refer to the install templates available in the source codes
|
7
7
|
|
8
|
-
ENV['
|
9
|
-
ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
|
8
|
+
ENV['KARAFKA_ENV'] ||= 'development'
|
10
9
|
Bundler.require(:default, ENV['KARAFKA_ENV'])
|
11
10
|
|
12
11
|
# Zeitwerk custom loader for loading the app components before the whole
|
@@ -10,7 +10,7 @@ module Karafka
|
|
10
10
|
# code here, as this is not a frequently used tracker. It is active only once per batch in
|
11
11
|
# case of long-running-jobs and upon errors.
|
12
12
|
class Pause < Base
|
13
|
-
attr_reader :attempt
|
13
|
+
attr_reader :attempt, :current_timeout
|
14
14
|
|
15
15
|
# @param timeout [Integer] how long should we wait when anything went wrong (in ms)
|
16
16
|
# @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
|
@@ -43,6 +43,7 @@ module Karafka
|
|
43
43
|
@started_at = nil
|
44
44
|
@attempt = 0
|
45
45
|
@timeout = timeout
|
46
|
+
@current_timeout = timeout
|
46
47
|
@max_timeout = max_timeout
|
47
48
|
@exponential_backoff = exponential_backoff
|
48
49
|
@mutex = Mutex.new
|
@@ -56,6 +57,7 @@ module Karafka
|
|
56
57
|
# period of time, outside of any regular pausing logic
|
57
58
|
def pause(timeout = backoff_interval)
|
58
59
|
@mutex.synchronize do
|
60
|
+
@current_timeout = timeout
|
59
61
|
@started_at = now
|
60
62
|
@ends_at = @started_at + timeout
|
61
63
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2022-12-
|
38
|
+
date: 2022-12-19 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 2.0.
|
46
|
+
version: 2.0.7
|
47
47
|
- - "<"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: 3.0.0
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
requirements:
|
54
54
|
- - ">="
|
55
55
|
- !ruby/object:Gem::Version
|
56
|
-
version: 2.0.
|
56
|
+
version: 2.0.7
|
57
57
|
- - "<"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: 3.0.0
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 2.4.
|
80
|
+
version: 2.4.7
|
81
81
|
- - "<"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: 3.0.0
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 2.4.
|
90
|
+
version: 2.4.7
|
91
91
|
- - "<"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: 3.0.0
|
@@ -147,7 +147,8 @@ files:
|
|
147
147
|
- bin/stress_one
|
148
148
|
- certs/cert_chain.pem
|
149
149
|
- certs/karafka-pro.pem
|
150
|
-
- config/errors.yml
|
150
|
+
- config/locales/errors.yml
|
151
|
+
- config/locales/pro_errors.yml
|
151
152
|
- docker-compose.yml
|
152
153
|
- karafka.gemspec
|
153
154
|
- lib/active_job/karafka.rb
|
@@ -204,6 +205,7 @@ files:
|
|
204
205
|
- lib/karafka/messages/message.rb
|
205
206
|
- lib/karafka/messages/messages.rb
|
206
207
|
- lib/karafka/messages/metadata.rb
|
208
|
+
- lib/karafka/messages/parser.rb
|
207
209
|
- lib/karafka/messages/seek.rb
|
208
210
|
- lib/karafka/patches/rdkafka/bindings.rb
|
209
211
|
- lib/karafka/patches/rdkafka/consumer.rb
|
@@ -212,6 +214,13 @@ files:
|
|
212
214
|
- lib/karafka/pro/active_job/dispatcher.rb
|
213
215
|
- lib/karafka/pro/active_job/job_options_contract.rb
|
214
216
|
- lib/karafka/pro/base_consumer.rb
|
217
|
+
- lib/karafka/pro/encryption.rb
|
218
|
+
- lib/karafka/pro/encryption/cipher.rb
|
219
|
+
- lib/karafka/pro/encryption/contracts/config.rb
|
220
|
+
- lib/karafka/pro/encryption/errors.rb
|
221
|
+
- lib/karafka/pro/encryption/messages/middleware.rb
|
222
|
+
- lib/karafka/pro/encryption/messages/parser.rb
|
223
|
+
- lib/karafka/pro/encryption/setup/config.rb
|
215
224
|
- lib/karafka/pro/loader.rb
|
216
225
|
- lib/karafka/pro/performance_tracker.rb
|
217
226
|
- lib/karafka/pro/processing/coordinator.rb
|
metadata.gz.sig
CHANGED
Binary file
|