karafka 2.0.27 → 2.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +34 -795
  4. data/Gemfile.lock +11 -10
  5. data/README.md +3 -2
  6. data/bin/karafka +1 -19
  7. data/bin/verify_license_integrity +2 -0
  8. data/config/locales/pro_errors.yml +1 -1
  9. data/karafka.gemspec +1 -1
  10. data/lib/karafka/cli/base.rb +20 -0
  11. data/lib/karafka/cli/console.rb +13 -8
  12. data/lib/karafka/connection/listener.rb +1 -1
  13. data/lib/karafka/instrumentation/callbacks/error.rb +1 -0
  14. data/lib/karafka/pro/active_job/consumer.rb +3 -2
  15. data/lib/karafka/pro/processing/collapser.rb +62 -0
  16. data/lib/karafka/pro/processing/coordinator.rb +17 -0
  17. data/lib/karafka/pro/processing/partitioner.rb +19 -5
  18. data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb +70 -0
  19. data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb +68 -0
  20. data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +1 -0
  21. data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +4 -0
  22. data/lib/karafka/pro/processing/strategies/default.rb +2 -2
  23. data/lib/karafka/pro/processing/strategies/dlq.rb +31 -7
  24. data/lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb +36 -0
  25. data/lib/karafka/pro/processing/strategies/dlq_vp.rb +37 -0
  26. data/lib/karafka/pro/processing/strategies/lrj_vp.rb +1 -0
  27. data/lib/karafka/pro/processing/strategies/vp.rb +5 -0
  28. data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +6 -5
  29. data/lib/karafka/processing/coordinator.rb +27 -8
  30. data/lib/karafka/processing/partitioner.rb +3 -1
  31. data/lib/karafka/processing/strategies/default.rb +2 -3
  32. data/lib/karafka/railtie.rb +2 -9
  33. data/lib/karafka/setup/config.rb +3 -1
  34. data/lib/karafka/templates/karafka.rb.erb +7 -1
  35. data/lib/karafka/version.rb +1 -1
  36. data/lib/karafka.rb +13 -0
  37. data.tar.gz.sig +0 -0
  38. metadata +9 -4
  39. metadata.gz.sig +0 -0
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This Karafka component is a Pro component under a commercial license.
4
+ # This Karafka component is NOT licensed under LGPL.
5
+ #
6
+ # All of the commercial components are present in the lib/karafka/pro directory of this
7
+ # repository and their usage requires commercial license agreement.
8
+ #
9
+ # Karafka has also commercial-friendly license, commercial support and commercial components.
10
+ #
11
+ # By sending a pull request to the pro components, you are agreeing to transfer the copyright of
12
+ # your code to Maciej Mensfeld.
13
+
14
+ module Karafka
15
+ module Pro
16
+ module Processing
17
+ module Strategies
18
+ # Dead Letter Queue enabled
19
+ # Virtual Partitions enabled
20
+ #
21
+ # In general because we collapse processing in virtual partitions to one on errors, there
22
+ # is no special action that needs to be taken because we warranty that even with VPs
23
+ # on errors a retry collapses into a single state.
24
+ module DlqVp
25
+ # Features for this strategy
26
+ FEATURES = %i[
27
+ dead_letter_queue
28
+ virtual_partitions
29
+ ].freeze
30
+
31
+ include Dlq
32
+ include Vp
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -19,6 +19,7 @@ module Karafka
19
19
  # Virtual Partitions enabled
20
20
  module LrjVp
21
21
  # Same flow as the standard Lrj
22
+ include Vp
22
23
  include Lrj
23
24
 
24
25
  # Features for this strategy
@@ -25,6 +25,11 @@ module Karafka
25
25
  FEATURES = %i[
26
26
  virtual_partitions
27
27
  ].freeze
28
+
29
+ # @return [Boolean] is the virtual processing collapsed in the context of given consumer.
30
+ def collapsed?
31
+ coordinator.collapsed?
32
+ end
28
33
  end
29
34
  end
30
35
  end
@@ -26,10 +26,10 @@ module Karafka
26
26
  ).fetch('en').fetch('validations').fetch('topic')
27
27
  end
28
28
 
29
- # Make sure that we don't use DLQ with VP
30
- # Using those two would cause many issues because the offset within VP is not
31
- # manageable so in scenarios where we would fail on the last message, we would move by
32
- # one and try again and fail, and move by one and try again and fail and so on...
29
+ # Make sure that when we use virtual partitions with DLQ, at least one retry is set
30
+ # We cannot use VP with DLQ without retries as we in order to provide ordering
31
+ # warranties on errors with VP, we need to collapse the VPs concurrency and retry
32
+ # without any indeterministic work
33
33
  virtual do |data, errors|
34
34
  next unless errors.empty?
35
35
 
@@ -38,8 +38,9 @@ module Karafka
38
38
 
39
39
  next unless dead_letter_queue[:active]
40
40
  next unless virtual_partitions[:active]
41
+ next if dead_letter_queue[:max_retries].positive?
41
42
 
42
- [[%i[dead_letter_queue], :not_with_virtual_partitions]]
43
+ [[%i[dead_letter_queue], :with_virtual_partitions]]
43
44
  end
44
45
  end
45
46
  end
@@ -71,18 +71,28 @@ module Karafka
71
71
  end
72
72
  end
73
73
 
74
- # @param consumer [Object] karafka consumer (normal or pro)
75
- # @return [Karafka::Processing::Result] result object which we can use to indicate
76
- # consumption processing state.
77
- def consumption(consumer)
74
+ # Is all the consumption done and finished successfully for this coordinator
75
+ def success?
78
76
  @mutex.synchronize do
79
- @consumptions[consumer] ||= Processing::Result.new
77
+ @running_jobs.zero? && @consumptions.values.all?(&:success?)
80
78
  end
81
79
  end
82
80
 
83
- # Is all the consumption done and finished successfully for this coordinator
84
- def success?
85
- @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
81
+ # Mark given consumption on consumer as successful
82
+ # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
83
+ def success!(consumer)
84
+ @mutex.synchronize do
85
+ consumption(consumer).success!
86
+ end
87
+ end
88
+
89
+ # Mark given consumption on consumer as failed
90
+ # @param consumer [Karafka::BaseConsumer] consumer that failed
91
+ # @param error [StandardError] error that occurred
92
+ def failure!(consumer, error)
93
+ @mutex.synchronize do
94
+ consumption(consumer).failure!(error)
95
+ end
86
96
  end
87
97
 
88
98
  # Marks given coordinator for processing group as revoked
@@ -113,6 +123,15 @@ module Karafka
113
123
  def manual_pause?
114
124
  @pause_tracker.paused? && @manual_pause
115
125
  end
126
+
127
+ private
128
+
129
+ # @param consumer [Object] karafka consumer (normal or pro)
130
+ # @return [Karafka::Processing::Result] result object which we can use to indicate
131
+ # consumption processing state.
132
+ def consumption(consumer)
133
+ @consumptions[consumer] ||= Processing::Result.new
134
+ end
116
135
  end
117
136
  end
118
137
  end
@@ -12,9 +12,11 @@ module Karafka
12
12
 
13
13
  # @param _topic [String] topic name
14
14
  # @param messages [Array<Karafka::Messages::Message>] karafka messages
15
+ # @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
16
+ # be used with those messages
15
17
  # @yieldparam [Integer] group id
16
18
  # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
17
- def call(_topic, messages)
19
+ def call(_topic, messages, _coordinator)
18
20
  yield(0, messages)
19
21
  end
20
22
  end
@@ -31,10 +31,9 @@ module Karafka
31
31
  end
32
32
 
33
33
  # Mark job as successful
34
- coordinator.consumption(self).success!
34
+ coordinator.success!(self)
35
35
  rescue StandardError => e
36
- # If failed, mark as failed
37
- coordinator.consumption(self).failure!(e)
36
+ coordinator.failure!(self, e)
38
37
 
39
38
  # Re-raise so reported in the consumer
40
39
  raise e
@@ -2,14 +2,7 @@
2
2
 
3
3
  # This file contains Railtie for auto-configuration
4
4
 
5
- rails = false
6
-
7
- begin
8
- # Do not load Rails again if already loaded
9
- Object.const_defined?('Rails::Railtie') || require('rails')
10
-
11
- rails = true
12
- rescue LoadError
5
+ unless Karafka.rails?
13
6
  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
14
7
  module Karafka
15
8
  class Railtie
@@ -17,7 +10,7 @@ rescue LoadError
17
10
  end
18
11
  end
19
12
 
20
- if rails
13
+ if Karafka.rails?
21
14
  # Load ActiveJob adapter
22
15
  require 'active_job/karafka'
23
16
 
@@ -16,7 +16,6 @@ module Karafka
16
16
 
17
17
  # Defaults for kafka settings, that will be overwritten only if not present already
18
18
  KAFKA_DEFAULTS = {
19
- 'client.id': 'karafka',
20
19
  # We emit the statistics by default, so all the instrumentation and web-ui work out of
21
20
  # the box, without requiring users to take any extra actions aside from enabling.
22
21
  'statistics.interval.ms': 5_000
@@ -193,6 +192,9 @@ module Karafka
193
192
  config.kafka[key] = value
194
193
  end
195
194
 
195
+ # Use Karafka client_id as kafka client id if not set
196
+ config.kafka[:'client.id'] ||= config.client_id
197
+
196
198
  return if Karafka::App.env.production?
197
199
 
198
200
  KAFKA_DEV_DEFAULTS.each do |key, value|
@@ -40,10 +40,16 @@ class KarafkaApp < Karafka::App
40
40
  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
41
41
  # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
42
42
 
43
+ # This logger prints the producer development info using the Karafka logger.
44
+ # It is similar to the consumer logger listener but producer oriented.
45
+ Karafka.producer.monitor.subscribe(
46
+ WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
47
+ )
48
+
43
49
  routes.draw do
44
50
  <% if rails? -%>
45
51
  # Uncomment this if you use Karafka with ActiveJob
46
- # You ned to define the topic per each queue name you use
52
+ # You need to define the topic per each queue name you use
47
53
  # active_job_topic :default
48
54
  <% end -%>
49
55
  topic :example do
@@ -3,5 +3,5 @@
3
3
  # Main module namespace
4
4
  module Karafka
5
5
  # Current Karafka version
6
- VERSION = '2.0.27'
6
+ VERSION = '2.0.29'
7
7
  end
data/lib/karafka.rb CHANGED
@@ -70,6 +70,19 @@ module Karafka
70
70
  App.config.license.token != false
71
71
  end
72
72
 
73
+ # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
74
+ # the console invocation to Rails
75
+ def rails?
76
+ return @rails if instance_variable_defined?('@rails')
77
+
78
+ # Do not load Rails again if already loaded
79
+ Object.const_defined?('Rails::Railtie') || require('rails')
80
+
81
+ @rails = true
82
+ rescue LoadError
83
+ @rails = false
84
+ end
85
+
73
86
  # @return [String] path to a default file that contains booting procedure etc
74
87
  # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
75
88
  # have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.27
4
+ version: 2.0.29
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
36
36
  MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
37
37
  -----END CERTIFICATE-----
38
- date: 2023-01-11 00:00:00.000000000 Z
38
+ date: 2023-01-30 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -77,7 +77,7 @@ dependencies:
77
77
  requirements:
78
78
  - - ">="
79
79
  - !ruby/object:Gem::Version
80
- version: 2.4.9
80
+ version: 2.4.10
81
81
  - - "<"
82
82
  - !ruby/object:Gem::Version
83
83
  version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
87
87
  requirements:
88
88
  - - ">="
89
89
  - !ruby/object:Gem::Version
90
- version: 2.4.9
90
+ version: 2.4.10
91
91
  - - "<"
92
92
  - !ruby/object:Gem::Version
93
93
  version: 3.0.0
@@ -224,6 +224,7 @@ files:
224
224
  - lib/karafka/pro/encryption/setup/config.rb
225
225
  - lib/karafka/pro/loader.rb
226
226
  - lib/karafka/pro/performance_tracker.rb
227
+ - lib/karafka/pro/processing/collapser.rb
227
228
  - lib/karafka/pro/processing/coordinator.rb
228
229
  - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
229
230
  - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
@@ -231,7 +232,9 @@ files:
231
232
  - lib/karafka/pro/processing/partitioner.rb
232
233
  - lib/karafka/pro/processing/scheduler.rb
233
234
  - lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
235
+ - lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb
234
236
  - lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
237
+ - lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb
235
238
  - lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
236
239
  - lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
237
240
  - lib/karafka/pro/processing/strategies/aj_mom.rb
@@ -241,7 +244,9 @@ files:
241
244
  - lib/karafka/pro/processing/strategies/dlq.rb
242
245
  - lib/karafka/pro/processing/strategies/dlq_lrj.rb
243
246
  - lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
247
+ - lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb
244
248
  - lib/karafka/pro/processing/strategies/dlq_mom.rb
249
+ - lib/karafka/pro/processing/strategies/dlq_vp.rb
245
250
  - lib/karafka/pro/processing/strategies/lrj.rb
246
251
  - lib/karafka/pro/processing/strategies/lrj_mom.rb
247
252
  - lib/karafka/pro/processing/strategies/lrj_vp.rb
metadata.gz.sig CHANGED
Binary file