karafka 2.4.8 → 2.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +0 -1
- data/CHANGELOG.md +8 -0
- data/Gemfile +8 -5
- data/Gemfile.lock +23 -14
- data/bin/integrations +5 -0
- data/certs/cert.pem +26 -0
- data/config/locales/errors.yml +4 -0
- data/config/locales/pro_errors.yml +17 -0
- data/karafka.gemspec +1 -1
- data/lib/karafka/admin.rb +42 -0
- data/lib/karafka/contracts/config.rb +2 -0
- data/lib/karafka/errors.rb +3 -2
- data/lib/karafka/pro/loader.rb +2 -1
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +16 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +5 -1
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +17 -1
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +17 -1
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +22 -6
- data/lib/karafka/pro/recurring_tasks/consumer.rb +105 -0
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +53 -0
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +41 -0
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +35 -0
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +87 -0
- data/lib/karafka/pro/recurring_tasks/errors.rb +34 -0
- data/lib/karafka/pro/recurring_tasks/executor.rb +152 -0
- data/lib/karafka/pro/recurring_tasks/listener.rb +38 -0
- data/lib/karafka/pro/recurring_tasks/matcher.rb +38 -0
- data/lib/karafka/pro/recurring_tasks/schedule.rb +63 -0
- data/lib/karafka/pro/recurring_tasks/serializer.rb +113 -0
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +52 -0
- data/lib/karafka/pro/recurring_tasks/task.rb +151 -0
- data/lib/karafka/pro/recurring_tasks.rb +87 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +130 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +28 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +40 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +27 -0
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +44 -0
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +25 -0
- data/lib/karafka/processing/strategies/dlq.rb +16 -2
- data/lib/karafka/processing/strategies/dlq_mom.rb +25 -6
- data/lib/karafka/processing/worker.rb +11 -1
- data/lib/karafka/railtie.rb +11 -22
- data/lib/karafka/routing/features/dead_letter_queue/config.rb +3 -0
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +1 -0
- data/lib/karafka/routing/features/dead_letter_queue/topic.rb +7 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +12 -0
- data/lib/karafka/routing/topic.rb +14 -0
- data/lib/karafka/setup/config.rb +3 -0
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +44 -24
- metadata.gz.sig +0 -0
- data/certs/cert_chain.pem +0 -26
data/lib/karafka/railtie.rb
CHANGED
@@ -65,28 +65,6 @@ if Karafka.rails?
|
|
65
65
|
app.config.autoload_paths += %w[app/consumers]
|
66
66
|
end
|
67
67
|
|
68
|
-
initializer 'karafka.configure_rails_code_reloader' do
|
69
|
-
# There are components that won't work with older Rails version, so we check it and
|
70
|
-
# provide a failover
|
71
|
-
rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
|
72
|
-
|
73
|
-
next unless Rails.env.development?
|
74
|
-
next unless ENV.key?('KARAFKA_CLI')
|
75
|
-
next unless rails6plus
|
76
|
-
|
77
|
-
# We can have many listeners, but it does not matter in which we will reload the code
|
78
|
-
# as long as all the consumers will be re-created as Rails reload is thread-safe
|
79
|
-
::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
|
80
|
-
# If consumer persistence is enabled, no reason to reload because we will still keep
|
81
|
-
# old consumer instances in memory.
|
82
|
-
next if Karafka::App.config.consumer_persistence
|
83
|
-
# Reload code each time there is a change in the code
|
84
|
-
next unless Rails.application.reloaders.any?(&:updated?)
|
85
|
-
|
86
|
-
Rails.application.reloader.reload!
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
68
|
initializer 'karafka.release_active_record_connections' do
|
91
69
|
rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
|
92
70
|
|
@@ -136,6 +114,17 @@ if Karafka.rails?
|
|
136
114
|
end
|
137
115
|
end
|
138
116
|
end
|
117
|
+
|
118
|
+
initializer 'karafka.configure_worker_external_executor' do |app|
|
119
|
+
app.config.after_initialize do
|
120
|
+
app_config = Karafka::App.config
|
121
|
+
|
122
|
+
# We need to wrap execution of the core user code with a wrapper in case of Rails, so
|
123
|
+
# the auto-reload works as expected
|
124
|
+
worker_job_call_wrapper = app_config.consumer_persistence ? app.executor : app.reloader
|
125
|
+
app_config.internal.processing.worker_job_call_wrapper = worker_job_call_wrapper
|
126
|
+
end
|
127
|
+
end
|
139
128
|
end
|
140
129
|
end
|
141
130
|
end
|
@@ -21,6 +21,9 @@ module Karafka
|
|
21
21
|
:dispatch_method,
|
22
22
|
# Should we use `#mark_as_consumed` or `#mark_as_consumed!` (in flows that mark)
|
23
23
|
:marking_method,
|
24
|
+
# Should we mark as consumed after dispatch or not. True for most cases, except MOM where
|
25
|
+
# it is on user to decide (false by default)
|
26
|
+
:mark_after_dispatch,
|
24
27
|
# Initialize with kwargs
|
25
28
|
keyword_init: true
|
26
29
|
) do
|
@@ -21,6 +21,7 @@ module Karafka
|
|
21
21
|
required(:independent) { |val| [true, false].include?(val) }
|
22
22
|
required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
|
23
23
|
required(:transactional) { |val| [true, false].include?(val) }
|
24
|
+
required(:mark_after_dispatch) { |val| [true, false, nil].include?(val) }
|
24
25
|
|
25
26
|
required(:dispatch_method) do |val|
|
26
27
|
%i[produce_async produce_sync].include?(val)
|
@@ -22,6 +22,9 @@ module Karafka
|
|
22
22
|
# whether dispatch on dlq should be sync or async (async by default)
|
23
23
|
# @param marking_method [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Describes
|
24
24
|
# whether marking on DLQ should be async or sync (async by default)
|
25
|
+
# @param mark_after_dispatch [Boolean, nil] Should we mark after dispatch. `nil` means
|
26
|
+
# that the default strategy approach to marking will be used. `true` or `false`
|
27
|
+
# overwrites the default
|
25
28
|
# @return [Config] defined config
|
26
29
|
def dead_letter_queue(
|
27
30
|
max_retries: DEFAULT_MAX_RETRIES,
|
@@ -29,7 +32,8 @@ module Karafka
|
|
29
32
|
independent: false,
|
30
33
|
transactional: true,
|
31
34
|
dispatch_method: :produce_async,
|
32
|
-
marking_method: :mark_as_consumed
|
35
|
+
marking_method: :mark_as_consumed,
|
36
|
+
mark_after_dispatch: nil
|
33
37
|
)
|
34
38
|
@dead_letter_queue ||= Config.new(
|
35
39
|
active: !topic.nil?,
|
@@ -38,7 +42,8 @@ module Karafka
|
|
38
42
|
independent: independent,
|
39
43
|
transactional: transactional,
|
40
44
|
dispatch_method: dispatch_method,
|
41
|
-
marking_method: marking_method
|
45
|
+
marking_method: marking_method,
|
46
|
+
mark_after_dispatch: mark_after_dispatch
|
42
47
|
)
|
43
48
|
end
|
44
49
|
|
@@ -19,6 +19,18 @@ module Karafka
|
|
19
19
|
nested :eofed do
|
20
20
|
required(:active) { |val| [true, false].include?(val) }
|
21
21
|
end
|
22
|
+
|
23
|
+
virtual do |data, errors|
|
24
|
+
next unless errors.empty?
|
25
|
+
|
26
|
+
eofed = data[:eofed]
|
27
|
+
|
28
|
+
next unless eofed[:active]
|
29
|
+
|
30
|
+
next if data[:kafka][:'enable.partition.eof']
|
31
|
+
|
32
|
+
[[%i[eofed kafka], :enable]]
|
33
|
+
end
|
22
34
|
end
|
23
35
|
end
|
24
36
|
end
|
@@ -54,6 +54,20 @@ module Karafka
|
|
54
54
|
RUBY
|
55
55
|
end
|
56
56
|
|
57
|
+
# Often users want to have the same basic cluster setup with small setting alterations
|
58
|
+
# This method allows us to do so by setting `inherit` to `true`. Whe inherit is enabled,
|
59
|
+
# settings will be merged with defaults.
|
60
|
+
#
|
61
|
+
# @param settings [Hash] kafka scope settings. If `:inherit` key is provided, it will
|
62
|
+
# instruct the assignment to merge with root level defaults
|
63
|
+
#
|
64
|
+
# @note It is set to `false` by default to preserve backwards compatibility
|
65
|
+
def kafka=(settings = {})
|
66
|
+
inherit = settings.delete(:inherit)
|
67
|
+
|
68
|
+
@kafka = inherit ? Karafka::App.config.kafka.merge(settings) : settings
|
69
|
+
end
|
70
|
+
|
57
71
|
# @return [String] name of subscription that will go to librdkafka
|
58
72
|
def subscription_name
|
59
73
|
name
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -285,6 +285,9 @@ module Karafka
|
|
285
285
|
setting :expansions_selector, default: Processing::ExpansionsSelector.new
|
286
286
|
# option [Class] executor class
|
287
287
|
setting :executor_class, default: Processing::Executor
|
288
|
+
# option worker_job_call_wrapper [Proc, false] callable object that will be used to wrap
|
289
|
+
# the worker execution of a job or false if no wrapper needed
|
290
|
+
setting :worker_job_call_wrapper, default: false
|
288
291
|
end
|
289
292
|
|
290
293
|
# Things related to operating on messages
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.4.
|
4
|
+
version: 2.4.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -12,30 +12,30 @@ cert_chain:
|
|
12
12
|
-----BEGIN CERTIFICATE-----
|
13
13
|
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
14
14
|
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
15
|
-
|
15
|
+
MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
|
16
16
|
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
17
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
|
18
|
+
PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
|
19
|
+
ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
|
20
|
+
Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
|
21
|
+
JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
|
22
|
+
T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
|
23
|
+
flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
|
24
|
+
vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
|
25
|
+
iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
|
26
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
|
27
|
+
sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
28
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
|
29
|
+
eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
|
30
|
+
Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
|
31
|
+
agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
|
32
|
+
TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
|
33
|
+
Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
|
34
|
+
oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
|
35
|
+
i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
|
36
|
+
ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2024-08-
|
38
|
+
date: 2024-08-23 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: base64
|
@@ -163,7 +163,7 @@ files:
|
|
163
163
|
- bin/stress_one
|
164
164
|
- bin/verify_license_integrity
|
165
165
|
- bin/wait_for_kafka
|
166
|
-
- certs/
|
166
|
+
- certs/cert.pem
|
167
167
|
- certs/karafka-pro.pem
|
168
168
|
- config/locales/errors.yml
|
169
169
|
- config/locales/pro_errors.yml
|
@@ -370,6 +370,20 @@ files:
|
|
370
370
|
- lib/karafka/pro/processing/strategies/vp/default.rb
|
371
371
|
- lib/karafka/pro/processing/strategy_selector.rb
|
372
372
|
- lib/karafka/pro/processing/subscription_groups_coordinator.rb
|
373
|
+
- lib/karafka/pro/recurring_tasks.rb
|
374
|
+
- lib/karafka/pro/recurring_tasks/consumer.rb
|
375
|
+
- lib/karafka/pro/recurring_tasks/contracts/config.rb
|
376
|
+
- lib/karafka/pro/recurring_tasks/contracts/task.rb
|
377
|
+
- lib/karafka/pro/recurring_tasks/deserializer.rb
|
378
|
+
- lib/karafka/pro/recurring_tasks/dispatcher.rb
|
379
|
+
- lib/karafka/pro/recurring_tasks/errors.rb
|
380
|
+
- lib/karafka/pro/recurring_tasks/executor.rb
|
381
|
+
- lib/karafka/pro/recurring_tasks/listener.rb
|
382
|
+
- lib/karafka/pro/recurring_tasks/matcher.rb
|
383
|
+
- lib/karafka/pro/recurring_tasks/schedule.rb
|
384
|
+
- lib/karafka/pro/recurring_tasks/serializer.rb
|
385
|
+
- lib/karafka/pro/recurring_tasks/setup/config.rb
|
386
|
+
- lib/karafka/pro/recurring_tasks/task.rb
|
373
387
|
- lib/karafka/pro/routing/features/active_job.rb
|
374
388
|
- lib/karafka/pro/routing/features/active_job/builder.rb
|
375
389
|
- lib/karafka/pro/routing/features/base.rb
|
@@ -434,6 +448,12 @@ files:
|
|
434
448
|
- lib/karafka/pro/routing/features/periodic_job/config.rb
|
435
449
|
- lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb
|
436
450
|
- lib/karafka/pro/routing/features/periodic_job/topic.rb
|
451
|
+
- lib/karafka/pro/routing/features/recurring_tasks.rb
|
452
|
+
- lib/karafka/pro/routing/features/recurring_tasks/builder.rb
|
453
|
+
- lib/karafka/pro/routing/features/recurring_tasks/config.rb
|
454
|
+
- lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb
|
455
|
+
- lib/karafka/pro/routing/features/recurring_tasks/proxy.rb
|
456
|
+
- lib/karafka/pro/routing/features/recurring_tasks/topic.rb
|
437
457
|
- lib/karafka/pro/routing/features/swarm.rb
|
438
458
|
- lib/karafka/pro/routing/features/swarm/config.rb
|
439
459
|
- lib/karafka/pro/routing/features/swarm/contracts/routing.rb
|
metadata.gz.sig
CHANGED
Binary file
|
data/certs/cert_chain.pem
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
-----BEGIN CERTIFICATE-----
|
2
|
-
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
3
|
-
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
4
|
-
MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
|
5
|
-
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
6
|
-
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
|
7
|
-
8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
|
8
|
-
sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
|
9
|
-
wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
|
10
|
-
b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
|
11
|
-
ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
|
12
|
-
5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
|
13
|
-
YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
|
14
|
-
kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
|
15
|
-
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
|
16
|
-
1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
17
|
-
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
|
18
|
-
bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
|
19
|
-
3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
|
20
|
-
kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
|
21
|
-
IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
|
22
|
-
fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
|
23
|
-
qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
|
24
|
-
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
25
|
-
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
26
|
-
-----END CERTIFICATE-----
|