karafka 2.0.27 → 2.0.29
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +34 -795
- data/Gemfile.lock +11 -10
- data/README.md +3 -2
- data/bin/karafka +1 -19
- data/bin/verify_license_integrity +2 -0
- data/config/locales/pro_errors.yml +1 -1
- data/karafka.gemspec +1 -1
- data/lib/karafka/cli/base.rb +20 -0
- data/lib/karafka/cli/console.rb +13 -8
- data/lib/karafka/connection/listener.rb +1 -1
- data/lib/karafka/instrumentation/callbacks/error.rb +1 -0
- data/lib/karafka/pro/active_job/consumer.rb +3 -2
- data/lib/karafka/pro/processing/collapser.rb +62 -0
- data/lib/karafka/pro/processing/coordinator.rb +17 -0
- data/lib/karafka/pro/processing/partitioner.rb +19 -5
- data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb +70 -0
- data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb +68 -0
- data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb +4 -0
- data/lib/karafka/pro/processing/strategies/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/dlq.rb +31 -7
- data/lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb +36 -0
- data/lib/karafka/pro/processing/strategies/dlq_vp.rb +37 -0
- data/lib/karafka/pro/processing/strategies/lrj_vp.rb +1 -0
- data/lib/karafka/pro/processing/strategies/vp.rb +5 -0
- data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb +6 -5
- data/lib/karafka/processing/coordinator.rb +27 -8
- data/lib/karafka/processing/partitioner.rb +3 -1
- data/lib/karafka/processing/strategies/default.rb +2 -3
- data/lib/karafka/railtie.rb +2 -9
- data/lib/karafka/setup/config.rb +3 -1
- data/lib/karafka/templates/karafka.rb.erb +7 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +13 -0
- data.tar.gz.sig +0 -0
- metadata +9 -4
- metadata.gz.sig +0 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# This Karafka component is a Pro component under a commercial license.
|
4
|
+
# This Karafka component is NOT licensed under LGPL.
|
5
|
+
#
|
6
|
+
# All of the commercial components are present in the lib/karafka/pro directory of this
|
7
|
+
# repository and their usage requires commercial license agreement.
|
8
|
+
#
|
9
|
+
# Karafka has also commercial-friendly license, commercial support and commercial components.
|
10
|
+
#
|
11
|
+
# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
|
12
|
+
# your code to Maciej Mensfeld.
|
13
|
+
|
14
|
+
module Karafka
|
15
|
+
module Pro
|
16
|
+
module Processing
|
17
|
+
module Strategies
|
18
|
+
# Dead Letter Queue enabled
|
19
|
+
# Virtual Partitions enabled
|
20
|
+
#
|
21
|
+
# In general because we collapse processing in virtual partitions to one on errors, there
|
22
|
+
# is no special action that needs to be taken because we warranty that even with VPs
|
23
|
+
# on errors a retry collapses into a single state.
|
24
|
+
module DlqVp
|
25
|
+
# Features for this strategy
|
26
|
+
FEATURES = %i[
|
27
|
+
dead_letter_queue
|
28
|
+
virtual_partitions
|
29
|
+
].freeze
|
30
|
+
|
31
|
+
include Dlq
|
32
|
+
include Vp
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -26,10 +26,10 @@ module Karafka
|
|
26
26
|
).fetch('en').fetch('validations').fetch('topic')
|
27
27
|
end
|
28
28
|
|
29
|
-
# Make sure that we
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
29
|
+
# Make sure that when we use virtual partitions with DLQ, at least one retry is set
|
30
|
+
# We cannot use VP with DLQ without retries as we in order to provide ordering
|
31
|
+
# warranties on errors with VP, we need to collapse the VPs concurrency and retry
|
32
|
+
# without any indeterministic work
|
33
33
|
virtual do |data, errors|
|
34
34
|
next unless errors.empty?
|
35
35
|
|
@@ -38,8 +38,9 @@ module Karafka
|
|
38
38
|
|
39
39
|
next unless dead_letter_queue[:active]
|
40
40
|
next unless virtual_partitions[:active]
|
41
|
+
next if dead_letter_queue[:max_retries].positive?
|
41
42
|
|
42
|
-
[[%i[dead_letter_queue], :
|
43
|
+
[[%i[dead_letter_queue], :with_virtual_partitions]]
|
43
44
|
end
|
44
45
|
end
|
45
46
|
end
|
@@ -71,18 +71,28 @@ module Karafka
|
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
74
|
-
#
|
75
|
-
|
76
|
-
# consumption processing state.
|
77
|
-
def consumption(consumer)
|
74
|
+
# Is all the consumption done and finished successfully for this coordinator
|
75
|
+
def success?
|
78
76
|
@mutex.synchronize do
|
79
|
-
@
|
77
|
+
@running_jobs.zero? && @consumptions.values.all?(&:success?)
|
80
78
|
end
|
81
79
|
end
|
82
80
|
|
83
|
-
#
|
84
|
-
|
85
|
-
|
81
|
+
# Mark given consumption on consumer as successful
|
82
|
+
# @param consumer [Karafka::BaseConsumer] consumer that finished successfully
|
83
|
+
def success!(consumer)
|
84
|
+
@mutex.synchronize do
|
85
|
+
consumption(consumer).success!
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Mark given consumption on consumer as failed
|
90
|
+
# @param consumer [Karafka::BaseConsumer] consumer that failed
|
91
|
+
# @param error [StandardError] error that occurred
|
92
|
+
def failure!(consumer, error)
|
93
|
+
@mutex.synchronize do
|
94
|
+
consumption(consumer).failure!(error)
|
95
|
+
end
|
86
96
|
end
|
87
97
|
|
88
98
|
# Marks given coordinator for processing group as revoked
|
@@ -113,6 +123,15 @@ module Karafka
|
|
113
123
|
def manual_pause?
|
114
124
|
@pause_tracker.paused? && @manual_pause
|
115
125
|
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
# @param consumer [Object] karafka consumer (normal or pro)
|
130
|
+
# @return [Karafka::Processing::Result] result object which we can use to indicate
|
131
|
+
# consumption processing state.
|
132
|
+
def consumption(consumer)
|
133
|
+
@consumptions[consumer] ||= Processing::Result.new
|
134
|
+
end
|
116
135
|
end
|
117
136
|
end
|
118
137
|
end
|
@@ -12,9 +12,11 @@ module Karafka
|
|
12
12
|
|
13
13
|
# @param _topic [String] topic name
|
14
14
|
# @param messages [Array<Karafka::Messages::Message>] karafka messages
|
15
|
+
# @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
|
16
|
+
# be used with those messages
|
15
17
|
# @yieldparam [Integer] group id
|
16
18
|
# @yieldparam [Array<Karafka::Messages::Message>] karafka messages
|
17
|
-
def call(_topic, messages)
|
19
|
+
def call(_topic, messages, _coordinator)
|
18
20
|
yield(0, messages)
|
19
21
|
end
|
20
22
|
end
|
@@ -31,10 +31,9 @@ module Karafka
|
|
31
31
|
end
|
32
32
|
|
33
33
|
# Mark job as successful
|
34
|
-
coordinator.
|
34
|
+
coordinator.success!(self)
|
35
35
|
rescue StandardError => e
|
36
|
-
|
37
|
-
coordinator.consumption(self).failure!(e)
|
36
|
+
coordinator.failure!(self, e)
|
38
37
|
|
39
38
|
# Re-raise so reported in the consumer
|
40
39
|
raise e
|
data/lib/karafka/railtie.rb
CHANGED
@@ -2,14 +2,7 @@
|
|
2
2
|
|
3
3
|
# This file contains Railtie for auto-configuration
|
4
4
|
|
5
|
-
rails
|
6
|
-
|
7
|
-
begin
|
8
|
-
# Do not load Rails again if already loaded
|
9
|
-
Object.const_defined?('Rails::Railtie') || require('rails')
|
10
|
-
|
11
|
-
rails = true
|
12
|
-
rescue LoadError
|
5
|
+
unless Karafka.rails?
|
13
6
|
# Without defining this in any way, Zeitwerk ain't happy so we do it that way
|
14
7
|
module Karafka
|
15
8
|
class Railtie
|
@@ -17,7 +10,7 @@ rescue LoadError
|
|
17
10
|
end
|
18
11
|
end
|
19
12
|
|
20
|
-
if rails
|
13
|
+
if Karafka.rails?
|
21
14
|
# Load ActiveJob adapter
|
22
15
|
require 'active_job/karafka'
|
23
16
|
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -16,7 +16,6 @@ module Karafka
|
|
16
16
|
|
17
17
|
# Defaults for kafka settings, that will be overwritten only if not present already
|
18
18
|
KAFKA_DEFAULTS = {
|
19
|
-
'client.id': 'karafka',
|
20
19
|
# We emit the statistics by default, so all the instrumentation and web-ui work out of
|
21
20
|
# the box, without requiring users to take any extra actions aside from enabling.
|
22
21
|
'statistics.interval.ms': 5_000
|
@@ -193,6 +192,9 @@ module Karafka
|
|
193
192
|
config.kafka[key] = value
|
194
193
|
end
|
195
194
|
|
195
|
+
# Use Karafka client_id as kafka client id if not set
|
196
|
+
config.kafka[:'client.id'] ||= config.client_id
|
197
|
+
|
196
198
|
return if Karafka::App.env.production?
|
197
199
|
|
198
200
|
KAFKA_DEV_DEFAULTS.each do |key, value|
|
@@ -40,10 +40,16 @@ class KarafkaApp < Karafka::App
|
|
40
40
|
Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
|
41
41
|
# Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
|
42
42
|
|
43
|
+
# This logger prints the producer development info using the Karafka logger.
|
44
|
+
# It is similar to the consumer logger listener but producer oriented.
|
45
|
+
Karafka.producer.monitor.subscribe(
|
46
|
+
WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
|
47
|
+
)
|
48
|
+
|
43
49
|
routes.draw do
|
44
50
|
<% if rails? -%>
|
45
51
|
# Uncomment this if you use Karafka with ActiveJob
|
46
|
-
# You
|
52
|
+
# You need to define the topic per each queue name you use
|
47
53
|
# active_job_topic :default
|
48
54
|
<% end -%>
|
49
55
|
topic :example do
|
data/lib/karafka/version.rb
CHANGED
data/lib/karafka.rb
CHANGED
@@ -70,6 +70,19 @@ module Karafka
|
|
70
70
|
App.config.license.token != false
|
71
71
|
end
|
72
72
|
|
73
|
+
# @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
|
74
|
+
# the console invocation to Rails
|
75
|
+
def rails?
|
76
|
+
return @rails if instance_variable_defined?('@rails')
|
77
|
+
|
78
|
+
# Do not load Rails again if already loaded
|
79
|
+
Object.const_defined?('Rails::Railtie') || require('rails')
|
80
|
+
|
81
|
+
@rails = true
|
82
|
+
rescue LoadError
|
83
|
+
@rails = false
|
84
|
+
end
|
85
|
+
|
73
86
|
# @return [String] path to a default file that contains booting procedure etc
|
74
87
|
# @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
|
75
88
|
# have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.29
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2023-01-
|
38
|
+
date: 2023-01-30 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: karafka-core
|
@@ -77,7 +77,7 @@ dependencies:
|
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version: 2.4.
|
80
|
+
version: 2.4.10
|
81
81
|
- - "<"
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: 3.0.0
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
requirements:
|
88
88
|
- - ">="
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: 2.4.
|
90
|
+
version: 2.4.10
|
91
91
|
- - "<"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: 3.0.0
|
@@ -224,6 +224,7 @@ files:
|
|
224
224
|
- lib/karafka/pro/encryption/setup/config.rb
|
225
225
|
- lib/karafka/pro/loader.rb
|
226
226
|
- lib/karafka/pro/performance_tracker.rb
|
227
|
+
- lib/karafka/pro/processing/collapser.rb
|
227
228
|
- lib/karafka/pro/processing/coordinator.rb
|
228
229
|
- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
|
229
230
|
- lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
|
@@ -231,7 +232,9 @@ files:
|
|
231
232
|
- lib/karafka/pro/processing/partitioner.rb
|
232
233
|
- lib/karafka/pro/processing/scheduler.rb
|
233
234
|
- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
|
235
|
+
- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb
|
234
236
|
- lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
|
237
|
+
- lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb
|
235
238
|
- lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
|
236
239
|
- lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
|
237
240
|
- lib/karafka/pro/processing/strategies/aj_mom.rb
|
@@ -241,7 +244,9 @@ files:
|
|
241
244
|
- lib/karafka/pro/processing/strategies/dlq.rb
|
242
245
|
- lib/karafka/pro/processing/strategies/dlq_lrj.rb
|
243
246
|
- lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
|
247
|
+
- lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb
|
244
248
|
- lib/karafka/pro/processing/strategies/dlq_mom.rb
|
249
|
+
- lib/karafka/pro/processing/strategies/dlq_vp.rb
|
245
250
|
- lib/karafka/pro/processing/strategies/lrj.rb
|
246
251
|
- lib/karafka/pro/processing/strategies/lrj_mom.rb
|
247
252
|
- lib/karafka/pro/processing/strategies/lrj_vp.rb
|
metadata.gz.sig
CHANGED
Binary file
|