karafka 2.0.23 → 2.0.26

Sign up to get free protection for your applications and to get access to all the features.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/workflows/ci.yml +24 -3
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +52 -1
  6. data/Gemfile.lock +14 -12
  7. data/README.md +6 -4
  8. data/bin/integrations +8 -0
  9. data/bin/verify_license_integrity +35 -0
  10. data/config/{errors.yml → locales/errors.yml} +2 -1
  11. data/config/locales/pro_errors.yml +18 -0
  12. data/docker-compose.yml +3 -0
  13. data/karafka.gemspec +3 -3
  14. data/lib/karafka/active_job/job_options_contract.rb +1 -1
  15. data/lib/karafka/admin.rb +16 -14
  16. data/lib/karafka/app.rb +16 -4
  17. data/lib/karafka/base_consumer.rb +37 -7
  18. data/lib/karafka/connection/client.rb +21 -0
  19. data/lib/karafka/connection/consumer_group_coordinator.rb +7 -1
  20. data/lib/karafka/connection/listener.rb +5 -4
  21. data/lib/karafka/connection/listeners_batch.rb +6 -0
  22. data/lib/karafka/contracts/config.rb +1 -1
  23. data/lib/karafka/contracts/consumer_group.rb +1 -1
  24. data/lib/karafka/contracts/server_cli_options.rb +2 -1
  25. data/lib/karafka/contracts/topic.rb +13 -2
  26. data/lib/karafka/instrumentation/logger_listener.rb +50 -2
  27. data/lib/karafka/instrumentation/notifications.rb +17 -7
  28. data/lib/karafka/instrumentation/proctitle_listener.rb +7 -16
  29. data/lib/karafka/instrumentation/vendors/datadog/listener.rb +2 -2
  30. data/lib/karafka/messages/message.rb +14 -2
  31. data/lib/karafka/messages/parser.rb +14 -0
  32. data/lib/karafka/pro/active_job/job_options_contract.rb +1 -1
  33. data/lib/karafka/pro/encryption/cipher.rb +58 -0
  34. data/lib/karafka/pro/encryption/contracts/config.rb +79 -0
  35. data/lib/karafka/pro/encryption/errors.rb +24 -0
  36. data/lib/karafka/pro/encryption/messages/middleware.rb +46 -0
  37. data/lib/karafka/pro/encryption/messages/parser.rb +56 -0
  38. data/lib/karafka/pro/encryption/setup/config.rb +48 -0
  39. data/lib/karafka/pro/encryption.rb +47 -0
  40. data/lib/karafka/pro/loader.rb +22 -1
  41. data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb +1 -1
  42. data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +6 -1
  43. data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +1 -1
  44. data/lib/karafka/pro/processing/strategies/default.rb +7 -1
  45. data/lib/karafka/pro/processing/strategies/dlq.rb +1 -1
  46. data/lib/karafka/pro/processing/strategies/dlq_lrj.rb +1 -1
  47. data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb +1 -1
  48. data/lib/karafka/pro/processing/strategies/dlq_mom.rb +1 -1
  49. data/lib/karafka/pro/processing/strategies/lrj.rb +6 -1
  50. data/lib/karafka/pro/processing/strategies/lrj_mom.rb +6 -1
  51. data/lib/karafka/pro/processing/strategies/mom.rb +1 -1
  52. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +2 -2
  53. data/lib/karafka/pro/routing/features/long_running_job/contract.rb +2 -2
  54. data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb +2 -2
  55. data/lib/karafka/process.rb +3 -1
  56. data/lib/karafka/processing/executor.rb +1 -1
  57. data/lib/karafka/processing/jobs_queue.rb +2 -2
  58. data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
  59. data/lib/karafka/processing/strategies/base.rb +5 -0
  60. data/lib/karafka/processing/strategies/default.rb +15 -1
  61. data/lib/karafka/processing/strategies/dlq.rb +1 -1
  62. data/lib/karafka/processing/strategies/dlq_mom.rb +1 -1
  63. data/lib/karafka/processing/strategies/mom.rb +1 -1
  64. data/lib/karafka/processing/worker.rb +3 -1
  65. data/lib/karafka/railtie.rb +3 -0
  66. data/lib/karafka/routing/builder.rb +1 -1
  67. data/lib/karafka/routing/consumer_group.rb +3 -3
  68. data/lib/karafka/routing/consumer_mapper.rb +0 -10
  69. data/lib/karafka/routing/features/active_job/contract.rb +1 -1
  70. data/lib/karafka/routing/features/dead_letter_queue/contract.rb +1 -1
  71. data/lib/karafka/routing/features/manual_offset_management/contract.rb +1 -1
  72. data/lib/karafka/routing/router.rb +12 -2
  73. data/lib/karafka/routing/subscription_group.rb +18 -1
  74. data/lib/karafka/routing/topic.rb +11 -0
  75. data/lib/karafka/runner.rb +1 -0
  76. data/lib/karafka/server.rb +27 -18
  77. data/lib/karafka/setup/config.rb +15 -2
  78. data/lib/karafka/status.rb +33 -9
  79. data/lib/karafka/templates/karafka.rb.erb +1 -2
  80. data/lib/karafka/time_trackers/base.rb +1 -6
  81. data/lib/karafka/time_trackers/pause.rb +5 -3
  82. data/lib/karafka/time_trackers/poll.rb +2 -2
  83. data/lib/karafka/version.rb +1 -1
  84. data/lib/karafka.rb +2 -0
  85. data.tar.gz.sig +0 -0
  86. metadata +18 -8
  87. metadata.gz.sig +0 -0
@@ -19,6 +19,7 @@ module Karafka
19
19
  # We aggregate threads here for a supervised shutdown process
20
20
  Karafka::Server.workers = workers
21
21
  Karafka::Server.listeners = listeners
22
+ Karafka::Server.jobs_queue = jobs_queue
22
23
 
23
24
  # All the listener threads need to finish
24
25
  listeners.each(&:join)
@@ -20,6 +20,9 @@ module Karafka
20
20
  # Set of workers
21
21
  attr_accessor :workers
22
22
 
23
+ # Jobs queue
24
+ attr_accessor :jobs_queue
25
+
23
26
  # Method which runs app
24
27
  def run
25
28
  self.listeners = []
@@ -45,9 +48,9 @@ module Karafka
45
48
 
46
49
  # We always need to wait for Karafka to stop here since we should wait for the stop running
47
50
  # in a separate thread (or trap context) to indicate everything is closed
48
- # Since `#start` is blocking, we were get here only after the runner is done. This will
51
+ # Since `#start` is blocking, we will get here only after the runner is done. This will
49
52
  # not add any performance degradation because of that.
50
- Thread.pass until Karafka::App.stopped?
53
+ sleep(0.1) until Karafka::App.terminated?
51
54
  # Try its best to shutdown underlying components before re-raising
52
55
  # rubocop:disable Lint/RescueException
53
56
  rescue Exception => e
@@ -75,6 +78,7 @@ module Karafka
75
78
  # Initialize the stopping process only if Karafka was running
76
79
  return if Karafka::App.stopping?
77
80
  return if Karafka::App.stopped?
81
+ return if Karafka::App.terminated?
78
82
 
79
83
  Karafka::App.stop!
80
84
 
@@ -84,13 +88,7 @@ module Karafka
84
88
  # their work and if so, we can just return and normal shutdown process will take place
85
89
  # We divide it by 1000 because we use time in ms.
86
90
  ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
87
- if listeners.count(&:alive?).zero? &&
88
- workers.count(&:alive?).zero?
89
-
90
- Karafka::App.producer.close
91
-
92
- return
93
- end
91
+ return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
94
92
 
95
93
  sleep SUPERVISION_SLEEP
96
94
  end
@@ -111,8 +109,6 @@ module Karafka
111
109
  # This can cause memory leaks and crashes.
112
110
  listeners.each(&:shutdown)
113
111
 
114
- Karafka::App.producer.close
115
-
116
112
  # We also do not forcefully terminate everything when running in the embedded mode,
117
113
  # otherwise we would overwrite the shutdown process of the process that started Karafka
118
114
  return unless process.supervised?
@@ -122,19 +118,32 @@ module Karafka
122
118
  ensure
123
119
  # We need to check if it wasn't an early exit to make sure that only on stop invocation
124
120
  # can change the status after everything is closed
125
- Karafka::App.stopped! if timeout
121
+ if timeout
122
+ Karafka::App.stopped!
123
+
124
+ # We close producer as the last thing as it can be used in the notification pipeline
125
+ # to dispatch state changes, etc
126
+ Karafka::App.producer.close
127
+
128
+ Karafka::App.terminate!
129
+ end
126
130
  end
127
131
 
128
132
  # Quiets the Karafka server.
129
- # Karafka will stop processing but won't quiet to consumer group, so no rebalance will be
133
+ #
134
+ # Karafka will stop processing but won't quit the consumer group, so no rebalance will be
130
135
  # triggered until final shutdown.
131
136
  def quiet
132
- # If we are already quieting or in the stop procedures, we should not do it again.
133
- return if Karafka::App.quieting?
134
- return if Karafka::App.stopping?
135
- return if Karafka::App.stopped?
136
-
137
+ # We don't have to safe-guard it with check states as the state transitions work only
138
+ # in one direction
137
139
  Karafka::App.quiet!
140
+
141
+ # We need one more thread to monitor the process and move to quieted once everything
142
+ # is quiet and no processing is happening anymore
143
+ Thread.new do
144
+ sleep(0.1) until listeners.coordinators.all?(&:finished?)
145
+ Karafka::App.quieted!
146
+ end
138
147
  end
139
148
 
140
149
  private
@@ -16,7 +16,10 @@ module Karafka
16
16
 
17
17
  # Defaults for kafka settings, that will be overwritten only if not present already
18
18
  KAFKA_DEFAULTS = {
19
- 'client.id': 'karafka'
19
+ 'client.id': 'karafka',
20
+ # We emit the statistics by default, so all the instrumentation and web-ui work out of
21
+ # the box, without requiring users to take any extra actions aside from enabling.
22
+ 'statistics.interval.ms': 5_000
20
23
  }.freeze
21
24
 
22
25
  # Contains settings that should not be used in production but make life easier in dev
@@ -130,6 +133,12 @@ module Karafka
130
133
  setting :strategy_selector, default: Processing::StrategySelector.new
131
134
  end
132
135
 
136
+ # Things related to operating on messages
137
+ setting :messages do
138
+ # Parser is used to convert raw payload prior to deserialization
139
+ setting :parser, default: Messages::Parser.new
140
+ end
141
+
133
142
  # Karafka components for ActiveJob
134
143
  setting :active_job do
135
144
  # option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
@@ -155,7 +164,7 @@ module Karafka
155
164
  # Will configure all the pro components
156
165
  # This needs to happen before end user configuration as the end user may overwrite some
157
166
  # of the pro defaults with custom components
158
- Pro::Loader.setup(config) if Karafka.pro?
167
+ Pro::Loader.pre_setup(config) if Karafka.pro?
159
168
 
160
169
  configure(&block)
161
170
  merge_kafka_defaults!(config)
@@ -164,6 +173,10 @@ module Karafka
164
173
 
165
174
  configure_components
166
175
 
176
+ # Runs things that need to be executed after config is defined and all the components
177
+ # are also configured
178
+ Pro::Loader.post_setup(config) if Karafka.pro?
179
+
167
180
  Karafka::App.initialized!
168
181
  end
169
182
 
@@ -8,34 +8,58 @@ module Karafka
8
8
  initializing: :initialize!,
9
9
  initialized: :initialized!,
10
10
  running: :run!,
11
+ # will no longer pickup any work, but current work will be finished
11
12
  quieting: :quiet!,
13
+ # no work is happening but we keep process with the assignments running
14
+ quiet: :quieted!,
15
+ # shutdown started
12
16
  stopping: :stop!,
13
- stopped: :stopped!
17
+ # all things are done and most of the things except critical are closed
18
+ stopped: :stopped!,
19
+ # immediately after this process exists
20
+ terminated: :terminate!
14
21
  }.freeze
15
22
 
16
- private_constant :STATES
23
+ # Mutex to ensure that state transitions are thread-safe
24
+ MUTEX = Mutex.new
25
+
26
+ private_constant :MUTEX
17
27
 
18
28
  # By default we are in the initializing state
19
29
  def initialize
20
30
  initialize!
21
31
  end
22
32
 
33
+ # @return [String] stringified current app status
34
+ def to_s
35
+ @status.to_s
36
+ end
37
+
38
+ # Resets the status state
39
+ # This is used mostly in the integration suite
40
+ def reset!
41
+ @status = :initializing
42
+ end
43
+
23
44
  STATES.each do |state, transition|
24
45
  define_method :"#{state}?" do
25
46
  @status == state
26
47
  end
27
48
 
28
49
  define_method transition do
29
- # Do nothing if the state change would change nothing (same state)
30
- return if @status == state
50
+ MUTEX.synchronize do
51
+ # Do not allow reverse state transitions (we always go one way) or transition to the same
52
+ # state as currently
53
+ return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
31
54
 
32
- @status = state
55
+ @status = state
33
56
 
34
- # Skip on creation (initializing)
35
- # We skip as during this state we do not have yet a monitor
36
- return if initializing?
57
+ # Skip on creation (initializing)
58
+ # We skip as during this state we do not have yet a monitor
59
+ return if initializing?
37
60
 
38
- Karafka.monitor.instrument("app.#{state}")
61
+ Karafka.monitor.instrument("app.#{state}")
62
+ end
39
63
  end
40
64
  end
41
65
  end
@@ -5,8 +5,7 @@
5
5
  # If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
6
6
  # command again or refer to the install templates available in the source codes
7
7
 
8
- ENV['RACK_ENV'] ||= 'development'
9
- ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
8
+ ENV['KARAFKA_ENV'] ||= 'development'
10
9
  Bundler.require(:default, ENV['KARAFKA_ENV'])
11
10
 
12
11
  # Zeitwerk custom loader for loading the app components before the whole
@@ -8,12 +8,7 @@ module Karafka
8
8
  module TimeTrackers
9
9
  # Base class for all the time-trackers.
10
10
  class Base
11
- private
12
-
13
- # @return [Float] current time in milliseconds
14
- def now
15
- ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000
16
- end
11
+ include ::Karafka::Core::Helpers::Time
17
12
  end
18
13
  end
19
14
  end
@@ -10,7 +10,7 @@ module Karafka
10
10
  # code here, as this is not a frequently used tracker. It is active only once per batch in
11
11
  # case of long-running-jobs and upon errors.
12
12
  class Pause < Base
13
- attr_reader :attempt
13
+ attr_reader :attempt, :current_timeout
14
14
 
15
15
  # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
16
16
  # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
@@ -43,6 +43,7 @@ module Karafka
43
43
  @started_at = nil
44
44
  @attempt = 0
45
45
  @timeout = timeout
46
+ @current_timeout = timeout
46
47
  @max_timeout = max_timeout
47
48
  @exponential_backoff = exponential_backoff
48
49
  @mutex = Mutex.new
@@ -56,7 +57,8 @@ module Karafka
56
57
  # period of time, outside of any regular pausing logic
57
58
  def pause(timeout = backoff_interval)
58
59
  @mutex.synchronize do
59
- @started_at = now
60
+ @current_timeout = timeout
61
+ @started_at = monotonic_now
60
62
  @ends_at = @started_at + timeout
61
63
  end
62
64
  end
@@ -93,7 +95,7 @@ module Karafka
93
95
  # @return [Boolean] did the pause expire
94
96
  def expired?
95
97
  @mutex.synchronize do
96
- @ends_at ? now >= @ends_at : true
98
+ @ends_at ? monotonic_now >= @ends_at : true
97
99
  end
98
100
  end
99
101
 
@@ -34,12 +34,12 @@ module Karafka
34
34
  # Starts time tracking.
35
35
  def start
36
36
  @attempts += 1
37
- @started_at = now
37
+ @started_at = monotonic_now
38
38
  end
39
39
 
40
40
  # Stops time tracking of a given piece of code and updates the remaining time.
41
41
  def checkpoint
42
- @remaining -= (now - @started_at)
42
+ @remaining -= (monotonic_now - @started_at)
43
43
  end
44
44
 
45
45
  # @return [Boolean] If anything went wrong, can we retry after a backoff period or not
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.0.23'
6
+ VERSION = '2.0.26'
7
7
  end
data/lib/karafka.rb CHANGED
@@ -14,7 +14,9 @@
14
14
  base64
15
15
  date
16
16
  singleton
17
+ digest
17
18
  zeitwerk
19
+ concurrent/atomic/atomic_fixnum
18
20
  ].each(&method(:require))
19
21
 
20
22
  # Karafka framework main namespace
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.23
4
+ version: 2.0.26
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
36
36
  MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
37
37
  -----END CERTIFICATE-----
38
- date: 2022-12-07 00:00:00.000000000 Z
38
+ date: 2023-01-10 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
43
43
  requirements:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
- version: 2.0.6
46
+ version: 2.0.8
47
47
  - - "<"
48
48
  - !ruby/object:Gem::Version
49
49
  version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
53
53
  requirements:
54
54
  - - ">="
55
55
  - !ruby/object:Gem::Version
56
- version: 2.0.6
56
+ version: 2.0.8
57
57
  - - "<"
58
58
  - !ruby/object:Gem::Version
59
59
  version: 3.0.0
@@ -77,7 +77,7 @@ dependencies:
77
77
  requirements:
78
78
  - - ">="
79
79
  - !ruby/object:Gem::Version
80
- version: 2.4.3
80
+ version: 2.4.7
81
81
  - - "<"
82
82
  - !ruby/object:Gem::Version
83
83
  version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
87
87
  requirements:
88
88
  - - ">="
89
89
  - !ruby/object:Gem::Version
90
- version: 2.4.3
90
+ version: 2.4.7
91
91
  - - "<"
92
92
  - !ruby/object:Gem::Version
93
93
  version: 3.0.0
@@ -145,9 +145,11 @@ files:
145
145
  - bin/scenario
146
146
  - bin/stress_many
147
147
  - bin/stress_one
148
+ - bin/verify_license_integrity
148
149
  - certs/cert_chain.pem
149
150
  - certs/karafka-pro.pem
150
- - config/errors.yml
151
+ - config/locales/errors.yml
152
+ - config/locales/pro_errors.yml
151
153
  - docker-compose.yml
152
154
  - karafka.gemspec
153
155
  - lib/active_job/karafka.rb
@@ -204,6 +206,7 @@ files:
204
206
  - lib/karafka/messages/message.rb
205
207
  - lib/karafka/messages/messages.rb
206
208
  - lib/karafka/messages/metadata.rb
209
+ - lib/karafka/messages/parser.rb
207
210
  - lib/karafka/messages/seek.rb
208
211
  - lib/karafka/patches/rdkafka/bindings.rb
209
212
  - lib/karafka/patches/rdkafka/consumer.rb
@@ -212,6 +215,13 @@ files:
212
215
  - lib/karafka/pro/active_job/dispatcher.rb
213
216
  - lib/karafka/pro/active_job/job_options_contract.rb
214
217
  - lib/karafka/pro/base_consumer.rb
218
+ - lib/karafka/pro/encryption.rb
219
+ - lib/karafka/pro/encryption/cipher.rb
220
+ - lib/karafka/pro/encryption/contracts/config.rb
221
+ - lib/karafka/pro/encryption/errors.rb
222
+ - lib/karafka/pro/encryption/messages/middleware.rb
223
+ - lib/karafka/pro/encryption/messages/parser.rb
224
+ - lib/karafka/pro/encryption/setup/config.rb
215
225
  - lib/karafka/pro/loader.rb
216
226
  - lib/karafka/pro/performance_tracker.rb
217
227
  - lib/karafka/pro/processing/coordinator.rb
@@ -340,7 +350,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
340
350
  - !ruby/object:Gem::Version
341
351
  version: '0'
342
352
  requirements: []
343
- rubygems_version: 3.3.26
353
+ rubygems_version: 3.4.1
344
354
  signing_key:
345
355
  specification_version: 4
346
356
  summary: Karafka is Ruby and Rails efficient Kafka processing framework.
metadata.gz.sig CHANGED
Binary file