karafka 2.0.27 → 2.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +34 -795
- data/Gemfile.lock +11 -10
- data/README.md +3 -2
- data/bin/karafka +1 -19
- data/bin/verify_license_integrity +2 -0
- data/config/locales/pro_errors.yml +1 -1
- data/karafka.gemspec +1 -1
- data/lib/karafka/cli/base.rb +20 -0
- data/lib/karafka/cli/console.rb +13 -8
- data/lib/karafka/connection/listener.rb +1 -1
- data/lib/karafka/instrumentation/callbacks/error.rb +1 -0
- data/lib/karafka/pro/active_job/consumer.rb +3 -2
- data/lib/karafka/pro/processing/collapser.rb +62 -0
- data/lib/karafka/pro/processing/coordinator.rb +17 -0
- data/lib/karafka/pro/processing/partitioner.rb +19 -5
- data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb +70 -0
- data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb +68 -0
- data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +4 -0
- data/lib/karafka/pro/processing/strategies/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/dlq.rb +31 -7
- data/lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb +36 -0
- data/lib/karafka/pro/processing/strategies/dlq_vp.rb +37 -0
- data/lib/karafka/pro/processing/strategies/lrj_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/vp.rb +5 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +6 -5
- data/lib/karafka/processing/coordinator.rb +27 -8
- data/lib/karafka/processing/partitioner.rb +3 -1
- data/lib/karafka/processing/strategies/default.rb +2 -3
- data/lib/karafka/railtie.rb +2 -9
- data/lib/karafka/setup/config.rb +3 -1
- data/lib/karafka/templates/karafka.rb.erb +7 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +13 -0
- data.tar.gz.sig +0 -0
- metadata +9 -4
- metadata.gz.sig +0 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
# Dead Letter Queue enabled
|
19
|
+
# Virtual Partitions enabled
|
20
|
+
#
|
21
|
+
# In general because we collapse processing in virtual partitions to one on errors, there
|
22
|
+
# is no special action that needs to be taken because we warranty that even with VPs
|
23
|
+
# on errors a retry collapses into a single state.
|
24
|
+
module DlqVp
|
25
|
+
# Features for this strategy
|
26
|
+
FEATURES = %i[
|
27
|
+
dead_letter_queue
|
28
|
+
virtual_partitions
|
29
|
+
].freeze
|
30
|
+
|
31
|
+
include Dlq
|
32
|
+
include Vp
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -26,10 +26,10 @@ module Karafka
|
|
26
26
|
).fetch('en').fetch('validations').fetch('topic')
|
27
27
|
end
|
28
28
|
|
29
|
-
# Make sure that we
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
29
|
+
# Make sure that when we use virtual partitions with DLQ, at least one retry is set
|
30
|
+
# We cannot use VP with DLQ without retries as we in order to provide ordering
|
31
|
+
# warranties on errors with VP, we need to collapse the VPs concurrency and retry
|
32
|
+
# without any indeterministic work
|
33
33
|
virtual do |data, errors|
|
34
34
|
next unless errors.empty?
|
35
35
|
|
@@ -38,8 +38,9 @@ module Karafka
|
|
38
38
|
|
39
39
|
next unless dead_letter_queue[:active]
|
40
40
|
next unless virtual_partitions[:active]
|
41
|
+
next if dead_letter_queue[:max_retries].positive?
|
41
42
|
|
42
|
-
[[%i[dead_letter_queue], :
|
43
|
+
[[%i[dead_letter_queue], :with_virtual_partitions]]
|
43
44
|
end
|
44
45
|
end
|
45
46
|
end
|
@@ -71,18 +71,28 @@ module Karafka
|
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
74
|
-
#
|
75
|
-
|
76
|
-
# consumption processing state.
|
77
|
-
def consumption(consumer)
|
74
|
+
# Is all the consumption done and finished successfully for this coordinator
|
75
|
+
def success?
|
78
76
|
@mutex.synchronize do
|
79
|
-
@
|
77
|
+
@running_jobs.zero? && @consumptions.values.all?(&:success?)
|
80
78
|
end
|
81
79
|
end
|
82
80
|
|
83
|
-
#
|
84
|
-
|
85
|
-
|
81
|
+
# Mark given consumption on consumer as successful
|
82
|
+
# @param consumer [Karafka::BaseConsumer] consumer that finished successfully
|
83
|
+
def success!(consumer)
|
84
|
+
@mutex.synchronize do
|
85
|
+
consumption(consumer).success!
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Mark given consumption on consumer as failed
|
90
|
+
# @param consumer [Karafka::BaseConsumer] consumer that failed
|
91
|
+
# @param error [StandardError] error that occurred
|
92
|
+
def failure!(consumer, error)
|
93
|
+
@mutex.synchronize do
|
94
|
+
consumption(consumer).failure!(error)
|
95
|
+
end
|
86
96
|
end
|
87
97
|
|
88
98
|
# Marks given coordinator for processing group as revoked
|
@@ -113,6 +123,15 @@ module Karafka
|
|
113
123
|
def manual_pause?
|
114
124
|
@pause_tracker.paused? && @manual_pause
|
115
125
|
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
# @param consumer [Object] karafka consumer (normal or pro)
|
130
|
+
# @return [Karafka::Processing::Result] result object which we can use to indicate
|
131
|
+
# consumption processing state.
|
132
|
+
def consumption(consumer)
|
133
|
+
@consumptions[consumer] ||= Processing::Result.new
|
134
|
+
end
|
116
135
|
end
|
117
136
|
end
|
118
137
|
end
|
@@ -12,9 +12,11 @@ module Karafka
|
|
12
12
|
|
13
13
|
# @param _topic [String] topic name
|
14
14
|
# @param messages [Array<Karafka::Messages::Message>] karafka messages
|
15
|
+
# @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
|
16
|
+
# be used with those messages
|
15
17
|
# @yieldparam [Integer] group id
|
16
18
|
# @yieldparam [Array<Karafka::Messages::Message>] karafka messages
|
17
|
-
def call(_topic, messages)
|
19
|
+
def call(_topic, messages, _coordinator)
|
18
20
|
yield(0, messages)
|
19
21
|
end
|
20
22
|
end
|
@@ -31,10 +31,9 @@ module Karafka
|
|
31
31
|
end
|
32
32
|
|
33
33
|
# Mark job as successful
|
34
|
-
coordinator.
|
34
|
+
coordinator.success!(self)
|
35
35
|
rescue StandardError => e
|
36
|
-
|
37
|
-
coordinator.consumption(self).failure!(e)
|
36
|
+
coordinator.failure!(self, e)
|
38
37
|
|
39
38
|
# Re-raise so reported in the consumer
|
40
39
|
raise e
|
data/lib/karafka/railtie.rb
CHANGED
@@ -2,14 +2,7 @@
|
|
2
2
|
|
3
3
|
# This file contains Railtie for auto-configuration
|
4
4
|
|
5
|
-
rails
|
6
|
-
|
7
|
-
begin
|
8
|
-
# Do not load Rails again if already loaded
|
9
|
-
Object.const_defined?('Rails::Railtie') || require('rails')
|
10
|
-
|
11
|
-
rails = true
|
12
|
-
rescue LoadError
|
5
|
+
unless Karafka.rails?
|
13
6
|
# Without defining this in any way, Zeitwerk ain't happy so we do it that way
|
14
7
|
module Karafka
|
15
8
|
class Railtie
|
@@ -17,7 +10,7 @@ rescue LoadError
|
|
17
10
|
end
|
18
11
|
end
|
19
12
|
|
20
|
-
if rails
|
13
|
+
if Karafka.rails?
|
21
14
|
# Load ActiveJob adapter
|
22
15
|
require 'active_job/karafka'
|
23
16
|
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -16,7 +16,6 @@ module Karafka
|
|
16
16
|
|
17
17
|
# Defaults for kafka settings, that will be overwritten only if not present already
|
18
18
|
KAFKA_DEFAULTS = {
|
19
|
-
'client.id': 'karafka',
|
20
19
|
# We emit the statistics by default, so all the instrumentation and web-ui work out of
|
21
20
|
# the box, without requiring users to take any extra actions aside from enabling.
|
22
21
|
'statistics.interval.ms': 5_000
|
@@ -193,6 +192,9 @@ module Karafka
|
|
193
192
|
config.kafka[key] = value
|
194
193
|
end
|
195
194
|
|
195
|
+
# Use Karafka client_id as kafka client id if not set
|
196
|
+
config.kafka[:'client.id'] ||= config.client_id
|
197
|
+
|
196
198
|
return if Karafka::App.env.production?
|
197
199
|
|
198
200
|
KAFKA_DEV_DEFAULTS.each do |key, value|
|
@@ -40,10 +40,16 @@ class KarafkaApp < Karafka::App
|
|
40
40
|
Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
|
41
41
|
# Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
|
42
42
|
|
43
|
+
# This logger prints the producer development info using the Karafka logger.
|
44
|
+
# It is similar to the consumer logger listener but producer oriented.
|
45
|
+
Karafka.producer.monitor.subscribe(
|
46
|
+
WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
|
47
|
+
)
|
48
|
+
|
43
49
|
routes.draw do
|
44
50
|
<% if rails? -%>
|
45
51
|
# Uncomment this if you use Karafka with ActiveJob
|
46
|
-
# You
|
52
|
+
# You need to define the topic per each queue name you use
|
47
53
|
# active_job_topic :default
|
48
54
|
<% end -%>
|
49
55
|
topic :example do
|
data/lib/karafka/version.rb
CHANGED
data/lib/karafka.rb
CHANGED
@@ -70,6 +70,19 @@ module Karafka
|
|
70
70
|
App.config.license.token != false
|
71
71
|
end
|
72
72
|
|
73
|
+
# @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
|
74
|
+
# the console invocation to Rails
|
75
|
+
def rails?
|
76
|
+
return @rails if instance_variable_defined?('@rails')
|
77
|
+
|
78
|
+
# Do not load Rails again if already loaded
|
79
|
+
Object.const_defined?('Rails::Railtie') || require('rails')
|
80
|
+
|
81
|
+
@rails = true
|
82
|
+
rescue LoadError
|
83
|
+
@rails = false
|
84
|
+
end
|
85
|
+
|
73
86
|
# @return [String] path to a default file that contains booting procedure etc
|
74
87
|
# @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
|
75
88
|
# have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.29
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-01-
|
38
|
+
date: 2023-01-30 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 2.4.
|
80
|
+
version: 2.4.10
|
81
81
|
- - "<"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: 3.0.0
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 2.4.
|
90
|
+
version: 2.4.10
|
91
91
|
- - "<"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: 3.0.0
|
@@ -224,6 +224,7 @@ files:
|
|
224
224
|
- lib/karafka/pro/encryption/setup/config.rb
|
225
225
|
- lib/karafka/pro/loader.rb
|
226
226
|
- lib/karafka/pro/performance_tracker.rb
|
227
|
+
- lib/karafka/pro/processing/collapser.rb
|
227
228
|
- lib/karafka/pro/processing/coordinator.rb
|
228
229
|
- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
|
229
230
|
- lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
|
@@ -231,7 +232,9 @@ files:
|
|
231
232
|
- lib/karafka/pro/processing/partitioner.rb
|
232
233
|
- lib/karafka/pro/processing/scheduler.rb
|
233
234
|
- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
|
235
|
+
- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb
|
234
236
|
- lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
|
237
|
+
- lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb
|
235
238
|
- lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
|
236
239
|
- lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
|
237
240
|
- lib/karafka/pro/processing/strategies/aj_mom.rb
|
@@ -241,7 +244,9 @@ files:
|
|
241
244
|
- lib/karafka/pro/processing/strategies/dlq.rb
|
242
245
|
- lib/karafka/pro/processing/strategies/dlq_lrj.rb
|
243
246
|
- lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
|
247
|
+
- lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb
|
244
248
|
- lib/karafka/pro/processing/strategies/dlq_mom.rb
|
249
|
+
- lib/karafka/pro/processing/strategies/dlq_vp.rb
|
245
250
|
- lib/karafka/pro/processing/strategies/lrj.rb
|
246
251
|
- lib/karafka/pro/processing/strategies/lrj_mom.rb
|
247
252
|
- lib/karafka/pro/processing/strategies/lrj_vp.rb
|
metadata.gz.sig
CHANGED
Binary file
|