karafka 2.5.1 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +3 -29
- data/.github/workflows/ci_macos_arm64.yml +1 -1
- data/.github/workflows/push.yml +2 -2
- data/.github/workflows/trigger-wiki-refresh.yml +1 -1
- data/.ruby-version +1 -1
- data/CHANGELOG.md +14 -4
- data/Gemfile +0 -2
- data/Gemfile.lock +30 -31
- data/bin/integrations +2 -1
- data/bin/rspecs +4 -0
- data/config/locales/errors.yml +6 -4
- data/config/locales/pro_errors.yml +5 -4
- data/docker-compose.yml +1 -1
- data/examples/payloads/json/sample_set_02/download.json +191 -0
- data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
- data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
- data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
- data/karafka.gemspec +1 -1
- data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
- data/lib/karafka/active_job/consumer.rb +5 -1
- data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
- data/lib/karafka/active_job/current_attributes.rb +1 -0
- data/lib/karafka/active_job/deserializer.rb +61 -0
- data/lib/karafka/active_job/dispatcher.rb +32 -12
- data/lib/karafka/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/admin/acl.rb +8 -4
- data/lib/karafka/admin/configs/config.rb +6 -4
- data/lib/karafka/admin/consumer_groups.rb +74 -4
- data/lib/karafka/admin/topics.rb +40 -7
- data/lib/karafka/admin.rb +13 -4
- data/lib/karafka/base_consumer.rb +5 -5
- data/lib/karafka/cli/base.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +2 -4
- data/lib/karafka/cli/install.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +1 -1
- data/lib/karafka/cli/topics/repartition.rb +2 -2
- data/lib/karafka/connection/client.rb +12 -2
- data/lib/karafka/connection/listeners_batch.rb +2 -3
- data/lib/karafka/connection/proxy.rb +11 -7
- data/lib/karafka/env.rb +1 -2
- data/lib/karafka/helpers/interval_runner.rb +4 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +17 -0
- data/lib/karafka/instrumentation/monitor.rb +1 -1
- data/lib/karafka/instrumentation/notifications.rb +1 -0
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +2 -3
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +2 -3
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +8 -9
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +2 -3
- data/lib/karafka/messages/builders/batch_metadata.rb +1 -1
- data/lib/karafka/messages/builders/message.rb +1 -1
- data/lib/karafka/messages/messages.rb +2 -3
- data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
- data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
- data/lib/karafka/pro/active_job/dispatcher.rb +7 -3
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
- data/lib/karafka/pro/cli/contracts/server.rb +2 -4
- data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +2 -2
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +2 -2
- data/lib/karafka/pro/connection/manager.rb +2 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +4 -6
- data/lib/karafka/pro/encryption/messages/parser.rb +3 -3
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
- data/lib/karafka/pro/iterator/expander.rb +1 -1
- data/lib/karafka/pro/iterator/tpl_builder.rb +1 -1
- data/lib/karafka/pro/iterator.rb +2 -2
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
- data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
- data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -4
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -4
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +6 -5
- data/lib/karafka/pro/recurring_tasks/schedule.rb +4 -6
- data/lib/karafka/pro/recurring_tasks.rb +8 -5
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
- data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -4
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +2 -4
- data/lib/karafka/pro/scheduled_messages.rb +4 -6
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/processing/coordinator.rb +2 -4
- data/lib/karafka/processing/coordinators_buffer.rb +2 -3
- data/lib/karafka/processing/executor.rb +2 -3
- data/lib/karafka/processing/jobs/base.rb +2 -3
- data/lib/karafka/processing/workers_batch.rb +2 -3
- data/lib/karafka/railtie.rb +1 -0
- data/lib/karafka/routing/activity_manager.rb +2 -2
- data/lib/karafka/routing/builder.rb +5 -7
- data/lib/karafka/routing/consumer_group.rb +4 -6
- data/lib/karafka/routing/contracts/consumer_group.rb +3 -5
- data/lib/karafka/routing/contracts/routing.rb +2 -4
- data/lib/karafka/routing/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/active_job/topic.rb +6 -0
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
- data/lib/karafka/routing/topics.rb +4 -9
- data/lib/karafka/server.rb +1 -1
- data/lib/karafka/setup/config.rb +66 -9
- data/lib/karafka/setup/contracts/config.rb +12 -10
- data/lib/karafka/setup/defaults_injector.rb +3 -2
- data/lib/karafka/setup/dsl.rb +2 -3
- data/lib/karafka/swarm/liveness_listener.rb +2 -3
- data/lib/karafka/swarm/supervisor.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +2 -2
- metadata +8 -2
- data/.diffend.yml +0 -3
|
@@ -13,6 +13,10 @@ module Karafka
|
|
|
13
13
|
# Extension allowing us to select correct subscriptions and assignments based on the
|
|
14
14
|
# expanded routing setup
|
|
15
15
|
module SubscriptionGroup
|
|
16
|
+
include Helpers::ConfigImporter.new(
|
|
17
|
+
swarm_node: %i[swarm node]
|
|
18
|
+
)
|
|
19
|
+
|
|
16
20
|
# @return [false, Array<String>] false if we do not have any subscriptions or array
|
|
17
21
|
# with all the subscriptions for given subscription group
|
|
18
22
|
def subscriptions
|
|
@@ -30,8 +34,7 @@ module Karafka
|
|
|
30
34
|
topics
|
|
31
35
|
.select(&:active?)
|
|
32
36
|
.select { |topic| topic.direct_assignments.active? }
|
|
33
|
-
.
|
|
34
|
-
.to_h
|
|
37
|
+
.to_h { |topic| build_assignments(topic) }
|
|
35
38
|
.tap { |topics| return false if topics.empty? }
|
|
36
39
|
.then { |topics| Iterator::Expander.new.call(topics) }
|
|
37
40
|
.then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
|
|
@@ -44,21 +47,19 @@ module Karafka
|
|
|
44
47
|
# @param topic [Karafka::Routing::Topic]
|
|
45
48
|
# @return [Array<String, Hash>]
|
|
46
49
|
def build_assignments(topic)
|
|
47
|
-
node = Karafka::App.config.swarm.node
|
|
48
|
-
|
|
49
50
|
standard_setup = [
|
|
50
51
|
topic.subscription_name,
|
|
51
52
|
topic.direct_assignments.partitions
|
|
52
53
|
]
|
|
53
54
|
|
|
54
|
-
return standard_setup unless
|
|
55
|
+
return standard_setup unless swarm_node
|
|
55
56
|
# Unless user explicitly assigned particular partitions to particular nodes, we just
|
|
56
57
|
# go with full regular assignments
|
|
57
58
|
return standard_setup unless topic.swarm.nodes.is_a?(Hash)
|
|
58
59
|
|
|
59
60
|
[
|
|
60
61
|
topic.subscription_name,
|
|
61
|
-
topic.swarm.nodes.fetch(
|
|
62
|
+
topic.swarm.nodes.fetch(swarm_node.id).to_h { |partition| [partition, true] }
|
|
62
63
|
]
|
|
63
64
|
end
|
|
64
65
|
end
|
|
@@ -43,12 +43,12 @@ module Karafka
|
|
|
43
43
|
|
|
44
44
|
Config.new(
|
|
45
45
|
active: true,
|
|
46
|
-
partitions: partitions_or_all.
|
|
46
|
+
partitions: partitions_or_all.to_h { |partition| [partition, true] }
|
|
47
47
|
)
|
|
48
48
|
else
|
|
49
49
|
Config.new(
|
|
50
50
|
active: !partitions_or_all.empty?,
|
|
51
|
-
partitions: partitions_or_all.
|
|
51
|
+
partitions: partitions_or_all.to_h { |partition| [partition, true] }
|
|
52
52
|
)
|
|
53
53
|
end
|
|
54
54
|
end
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the expiring feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the filtering feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -27,10 +27,9 @@ module Karafka
|
|
|
27
27
|
@filtering
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
# @param args [Array] Anything `#filter` accepts
|
|
31
30
|
# @return [Filtering::Config] alias to match the naming API for features
|
|
32
|
-
def filtering(*
|
|
33
|
-
filter(*
|
|
31
|
+
def filtering(*)
|
|
32
|
+
filter(*)
|
|
34
33
|
end
|
|
35
34
|
|
|
36
35
|
# @return [Boolean] is a given job throttled
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract for inline insights topic setup
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around long-running job settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -17,10 +17,8 @@ module Karafka
|
|
|
17
17
|
# multiplexing attributes are optional since multiplexing may not be enabled
|
|
18
18
|
class Topic < Karafka::Contracts::Base
|
|
19
19
|
configure do |config|
|
|
20
|
-
config.error_messages = YAML.
|
|
21
|
-
File.
|
|
22
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
23
|
-
)
|
|
20
|
+
config.error_messages = YAML.safe_load_file(
|
|
21
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
24
22
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
25
23
|
end
|
|
26
24
|
|
|
@@ -56,7 +54,7 @@ module Karafka
|
|
|
56
54
|
max = max(data)
|
|
57
55
|
boot = boot(data)
|
|
58
56
|
|
|
59
|
-
next if boot
|
|
57
|
+
next if boot.between?(min, max)
|
|
60
58
|
|
|
61
59
|
[[%w[subscription_group_details], :multiplexing_boot_mismatch]]
|
|
62
60
|
end
|
|
@@ -10,9 +10,9 @@ module Karafka
|
|
|
10
10
|
class NonBlockingJob < Base
|
|
11
11
|
# Non-Blocking Jobs topic API extensions
|
|
12
12
|
module Topic
|
|
13
|
-
#
|
|
14
|
-
def non_blocking_job(*
|
|
15
|
-
long_running_job(*
|
|
13
|
+
# Non blocking setting method
|
|
14
|
+
def non_blocking_job(*)
|
|
15
|
+
long_running_job(*)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
alias non_blocking non_blocking_job
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the expiring feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the parallel segments feature
|
|
14
14
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
|
|
21
19
|
|
|
22
20
|
nested(:parallel_segments) do
|
|
@@ -12,13 +12,11 @@ module Karafka
|
|
|
12
12
|
# Contract to validate configuration of the filtering feature
|
|
13
13
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
14
14
|
configure do |config|
|
|
15
|
-
config.error_messages = YAML.
|
|
16
|
-
File.
|
|
17
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
18
|
-
)
|
|
15
|
+
config.error_messages = YAML.safe_load_file(
|
|
16
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
17
|
).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
|
|
20
18
|
|
|
21
|
-
required(:patterns) { |val| val.is_a?(Array) && val.all?
|
|
19
|
+
required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
|
|
22
20
|
|
|
23
21
|
virtual do |data, errors|
|
|
24
22
|
next unless errors.empty?
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract used to validate pattern data
|
|
14
14
|
class Pattern < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('pattern')
|
|
21
19
|
|
|
22
20
|
required(:regexp) { |val| val.is_a?(Regexp) }
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the patterns feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
|
|
4
|
+
# See LICENSE for details.
|
|
5
|
+
|
|
6
|
+
module Karafka
|
|
7
|
+
module Pro
|
|
8
|
+
module Routing
|
|
9
|
+
module Features
|
|
10
|
+
class Pausing < Base
|
|
11
|
+
# Config for pausing feature
|
|
12
|
+
Config = Struct.new(
|
|
13
|
+
:active,
|
|
14
|
+
:timeout,
|
|
15
|
+
:max_timeout,
|
|
16
|
+
:with_exponential_backoff,
|
|
17
|
+
keyword_init: true
|
|
18
|
+
) do
|
|
19
|
+
alias_method :active?, :active
|
|
20
|
+
alias_method :with_exponential_backoff?, :with_exponential_backoff
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -13,25 +13,31 @@ module Karafka
|
|
|
13
13
|
# Contract to make sure, that the pause settings on a per topic basis are as expected
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
# Validate the nested pausing configuration
|
|
21
|
+
# Both old setters and new pause() method update the pausing config,
|
|
22
|
+
# so we only need to validate this one format
|
|
23
|
+
nested :pausing do
|
|
24
|
+
required(:active) { |val| [true, false].include?(val) }
|
|
25
|
+
required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
26
|
+
required(:max_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
27
|
+
required(:with_exponential_backoff) { |val| [true, false].include?(val) }
|
|
28
|
+
end
|
|
25
29
|
|
|
30
|
+
# Validate that timeout <= max_timeout
|
|
26
31
|
virtual do |data, errors|
|
|
27
32
|
next unless errors.empty?
|
|
28
33
|
|
|
29
|
-
|
|
30
|
-
|
|
34
|
+
pausing = data.fetch(:pausing)
|
|
35
|
+
timeout = pausing.fetch(:timeout)
|
|
36
|
+
max_timeout = pausing.fetch(:max_timeout)
|
|
31
37
|
|
|
32
|
-
next if
|
|
38
|
+
next if timeout <= max_timeout
|
|
33
39
|
|
|
34
|
-
[[%i[
|
|
40
|
+
[[%i[pausing timeout], :max_timeout_vs_pause_max_timeout]]
|
|
35
41
|
end
|
|
36
42
|
end
|
|
37
43
|
end
|
|
@@ -10,23 +10,84 @@ module Karafka
|
|
|
10
10
|
class Pausing < Base
|
|
11
11
|
# Expansion allowing for a per topic pause strategy definitions
|
|
12
12
|
module Topic
|
|
13
|
+
# This method calls the parent class initializer and then sets up the
|
|
14
|
+
# extra instance variable to nil. The explicit initialization
|
|
15
|
+
# to nil is included as an optimization for Ruby's object shapes system,
|
|
16
|
+
# which improves memory layout and access performance.
|
|
17
|
+
def initialize(...)
|
|
18
|
+
super
|
|
19
|
+
@pausing = nil
|
|
20
|
+
end
|
|
21
|
+
|
|
13
22
|
# Allows for per-topic pausing strategy setting
|
|
14
23
|
#
|
|
15
24
|
# @param timeout [Integer] how long should we wait upon processing error (milliseconds)
|
|
16
25
|
# @param max_timeout [Integer] what is the max timeout in case of an exponential
|
|
17
26
|
# backoff (milliseconds)
|
|
18
27
|
# @param with_exponential_backoff [Boolean] should we use exponential backoff
|
|
19
|
-
#
|
|
20
|
-
# @note We do not construct here the nested config like we do with other routing
|
|
21
|
-
# features, because this feature operates on the OSS layer by injection of values
|
|
22
|
-
# and a nested config is not needed.
|
|
28
|
+
# @return [Config] pausing config object
|
|
23
29
|
def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
|
|
24
|
-
|
|
25
|
-
|
|
30
|
+
# If no arguments provided, just return or initialize the config
|
|
31
|
+
return pausing if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
|
|
32
|
+
|
|
33
|
+
# Update instance variables for backwards compatibility
|
|
34
|
+
# This ensures code reading @pause_timeout directly or via the inherited getter
|
|
35
|
+
# will get the correct values
|
|
36
|
+
@pause_timeout = timeout if timeout
|
|
37
|
+
@pause_max_timeout = max_timeout if max_timeout
|
|
38
|
+
|
|
39
|
+
unless with_exponential_backoff.nil?
|
|
40
|
+
@pause_with_exponential_backoff = with_exponential_backoff
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Create or update the config
|
|
44
|
+
@pausing ||= Config.new(
|
|
45
|
+
active: false,
|
|
46
|
+
timeout: @pause_timeout || Karafka::App.config.pause.timeout,
|
|
47
|
+
max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
|
|
48
|
+
with_exponential_backoff: if @pause_with_exponential_backoff.nil?
|
|
49
|
+
Karafka::App.config.pause.with_exponential_backoff
|
|
50
|
+
else
|
|
51
|
+
@pause_with_exponential_backoff
|
|
52
|
+
end
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
@pausing.timeout = timeout if timeout
|
|
56
|
+
@pausing.max_timeout = max_timeout if max_timeout
|
|
26
57
|
|
|
27
|
-
|
|
58
|
+
unless with_exponential_backoff.nil?
|
|
59
|
+
@pausing.with_exponential_backoff = with_exponential_backoff
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
@pausing.active = true
|
|
63
|
+
|
|
64
|
+
@pausing
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# @return [Config] pausing configuration object
|
|
68
|
+
def pausing
|
|
69
|
+
@pausing ||= Config.new(
|
|
70
|
+
active: false,
|
|
71
|
+
timeout: @pause_timeout || Karafka::App.config.pause.timeout,
|
|
72
|
+
max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
|
|
73
|
+
with_exponential_backoff: if @pause_with_exponential_backoff.nil?
|
|
74
|
+
Karafka::App.config.pause.with_exponential_backoff
|
|
75
|
+
else
|
|
76
|
+
@pause_with_exponential_backoff
|
|
77
|
+
end
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @return [Boolean] is pausing explicitly configured
|
|
82
|
+
def pausing?
|
|
83
|
+
pausing.active?
|
|
84
|
+
end
|
|
28
85
|
|
|
29
|
-
|
|
86
|
+
# @return [Hash] topic with all its native configuration options plus pausing settings
|
|
87
|
+
def to_h
|
|
88
|
+
super.merge(
|
|
89
|
+
pausing: pausing.to_h
|
|
90
|
+
).freeze
|
|
30
91
|
end
|
|
31
92
|
end
|
|
32
93
|
end
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the periodics feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around recurring tasks settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around scheduled messages settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -18,10 +18,8 @@ module Karafka
|
|
|
18
18
|
# This is why we use it before warmup when everything is expected to be configured.
|
|
19
19
|
class Routing < Karafka::Contracts::Base
|
|
20
20
|
configure do |config|
|
|
21
|
-
config.error_messages = YAML.
|
|
22
|
-
File.
|
|
23
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
24
|
-
)
|
|
21
|
+
config.error_messages = YAML.safe_load_file(
|
|
22
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
25
23
|
).fetch('en').fetch('validations').fetch('routing')
|
|
26
24
|
end
|
|
27
25
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the swarm feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -25,15 +23,15 @@ module Karafka
|
|
|
25
23
|
|
|
26
24
|
required(:nodes) do |val|
|
|
27
25
|
next true if val.is_a?(Range)
|
|
28
|
-
next true if val.is_a?(Array) && val.all?
|
|
26
|
+
next true if val.is_a?(Array) && val.all?(Integer)
|
|
29
27
|
next false unless val.is_a?(Hash)
|
|
30
|
-
next false unless val.keys.all?
|
|
28
|
+
next false unless val.keys.all?(Integer)
|
|
31
29
|
|
|
32
30
|
values = val.values
|
|
33
31
|
|
|
34
32
|
next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
|
|
35
|
-
next true if values.flatten.all?
|
|
36
|
-
next true if values.flatten.all?
|
|
33
|
+
next true if values.flatten.all?(Integer)
|
|
34
|
+
next true if values.flatten.all?(Range)
|
|
37
35
|
|
|
38
36
|
false
|
|
39
37
|
end
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around throttling settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around virtual partitions
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -11,10 +11,8 @@ module Karafka
|
|
|
11
11
|
# Makes sure, all the expected config is defined as it should be
|
|
12
12
|
class Config < ::Karafka::Contracts::Base
|
|
13
13
|
configure do |config|
|
|
14
|
-
config.error_messages = YAML.
|
|
15
|
-
File.
|
|
16
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
17
|
-
)
|
|
14
|
+
config.error_messages = YAML.safe_load_file(
|
|
15
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
18
16
|
).fetch('en').fetch('validations').fetch('setup').fetch('config')
|
|
19
17
|
end
|
|
20
18
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# details to be able to dispatch the message
|
|
14
14
|
class Message < ::Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('scheduled_messages_message')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -24,17 +24,15 @@ module Karafka
|
|
|
24
24
|
|
|
25
25
|
class << self
|
|
26
26
|
# Runs the `Proxy.call`
|
|
27
|
-
# @param kwargs [Hash] things requested by the proxy
|
|
28
27
|
# @return [Hash] message wrapped with the scheduled message envelope
|
|
29
|
-
def schedule(**
|
|
30
|
-
Proxy.schedule(**
|
|
28
|
+
def schedule(**)
|
|
29
|
+
Proxy.schedule(**)
|
|
31
30
|
end
|
|
32
31
|
|
|
33
32
|
# Generates a tombstone message to cancel given dispatch (if not yet happened)
|
|
34
|
-
# @param kwargs [Hash] things requested by the proxy
|
|
35
33
|
# @return [Hash] tombstone cancelling message
|
|
36
|
-
def cancel(**
|
|
37
|
-
Proxy.cancel(**
|
|
34
|
+
def cancel(**)
|
|
35
|
+
Proxy.cancel(**)
|
|
38
36
|
end
|
|
39
37
|
|
|
40
38
|
# Below are private APIs
|
|
@@ -171,14 +171,14 @@ module Karafka
|
|
|
171
171
|
def rss_mb_linux
|
|
172
172
|
kb_rss = 0
|
|
173
173
|
|
|
174
|
-
|
|
174
|
+
File.readlines("/proc/#{node.pid}/status").each do |line|
|
|
175
175
|
next unless line.start_with?('VmRSS:')
|
|
176
176
|
|
|
177
177
|
kb_rss = line.split[1].to_i
|
|
178
178
|
break
|
|
179
179
|
end
|
|
180
180
|
|
|
181
|
-
(kb_rss / 1_024
|
|
181
|
+
(kb_rss / 1_024).round
|
|
182
182
|
rescue Errno::ENOENT, Errno::EACCES
|
|
183
183
|
# /proc file doesn't exist or no permission to read it
|
|
184
184
|
0
|
|
@@ -198,8 +198,6 @@ module Karafka
|
|
|
198
198
|
|
|
199
199
|
# Allows to run synchronized (locked) code that can operate only from a given thread
|
|
200
200
|
#
|
|
201
|
-
# @param block [Proc] code we want to run in the synchronized mode
|
|
202
|
-
#
|
|
203
201
|
# @note We check if mutex is not owned already by the current thread so we won't end up with
|
|
204
202
|
# a deadlock in case user runs coordinated code from inside of his own lock
|
|
205
203
|
#
|
|
@@ -207,11 +205,11 @@ module Karafka
|
|
|
207
205
|
# Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
|
|
208
206
|
# This can and should however be used for multi-thread strategy applications and other
|
|
209
207
|
# internal operations locks.
|
|
210
|
-
def synchronize(&
|
|
208
|
+
def synchronize(&)
|
|
211
209
|
if @mutex.owned?
|
|
212
210
|
yield
|
|
213
211
|
else
|
|
214
|
-
@mutex.synchronize(&
|
|
212
|
+
@mutex.synchronize(&)
|
|
215
213
|
end
|
|
216
214
|
end
|
|
217
215
|
end
|