karafka 2.5.1 → 2.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +21 -29
- data/.github/workflows/ci_macos_arm64.yml +1 -1
- data/.github/workflows/push.yml +2 -2
- data/.github/workflows/trigger-wiki-refresh.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +174 -0
- data/CHANGELOG.md +20 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +45 -41
- data/bin/integrations +2 -1
- data/bin/rspecs +4 -0
- data/config/locales/errors.yml +6 -4
- data/config/locales/pro_errors.yml +5 -4
- data/docker-compose.yml +1 -1
- data/examples/payloads/json/sample_set_02/download.json +191 -0
- data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
- data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
- data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
- data/karafka.gemspec +3 -3
- data/lib/active_job/queue_adapters/karafka_adapter.rb +3 -3
- data/lib/karafka/active_job/consumer.rb +7 -3
- data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
- data/lib/karafka/active_job/current_attributes.rb +3 -2
- data/lib/karafka/active_job/deserializer.rb +61 -0
- data/lib/karafka/active_job/dispatcher.rb +34 -14
- data/lib/karafka/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/admin/acl.rb +8 -4
- data/lib/karafka/admin/configs/config.rb +6 -4
- data/lib/karafka/admin/configs/resource.rb +7 -1
- data/lib/karafka/admin/consumer_groups.rb +80 -12
- data/lib/karafka/admin/topics.rb +43 -9
- data/lib/karafka/admin.rb +23 -14
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +6 -6
- data/lib/karafka/cli/base.rb +2 -2
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +3 -5
- data/lib/karafka/cli/help.rb +1 -1
- data/lib/karafka/cli/install.rb +3 -2
- data/lib/karafka/cli/server.rb +1 -1
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +1 -1
- data/lib/karafka/cli/topics/repartition.rb +2 -2
- data/lib/karafka/connection/client.rb +30 -19
- data/lib/karafka/connection/listeners_batch.rb +2 -3
- data/lib/karafka/connection/manager.rb +1 -0
- data/lib/karafka/connection/proxy.rb +12 -8
- data/lib/karafka/connection/rebalance_manager.rb +1 -1
- data/lib/karafka/connection/status.rb +1 -0
- data/lib/karafka/constraints.rb +1 -1
- data/lib/karafka/contracts/base.rb +1 -1
- data/lib/karafka/deserializers/payload.rb +1 -1
- data/lib/karafka/env.rb +1 -2
- data/lib/karafka/helpers/async.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +3 -3
- data/lib/karafka/helpers/interval_runner.rb +4 -1
- data/lib/karafka/helpers/multi_delegator.rb +3 -0
- data/lib/karafka/instrumentation/assignments_tracker.rb +19 -1
- data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
- data/lib/karafka/instrumentation/logger.rb +6 -6
- data/lib/karafka/instrumentation/monitor.rb +3 -3
- data/lib/karafka/instrumentation/notifications.rb +1 -0
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +3 -4
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +3 -4
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +10 -11
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +5 -18
- data/lib/karafka/messages/builders/batch_metadata.rb +2 -2
- data/lib/karafka/messages/builders/message.rb +1 -1
- data/lib/karafka/messages/messages.rb +2 -3
- data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
- data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
- data/lib/karafka/pro/active_job/consumer.rb +2 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +10 -6
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
- data/lib/karafka/pro/cleaner.rb +3 -3
- data/lib/karafka/pro/cli/contracts/server.rb +3 -5
- data/lib/karafka/pro/cli/parallel_segments/base.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
- data/lib/karafka/pro/connection/manager.rb +3 -4
- data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
- data/lib/karafka/pro/contracts/base.rb +1 -1
- data/lib/karafka/pro/encryption/cipher.rb +3 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +5 -7
- data/lib/karafka/pro/encryption/messages/parser.rb +4 -4
- data/lib/karafka/pro/encryption/setup/config.rb +1 -1
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
- data/lib/karafka/pro/iterator/expander.rb +1 -1
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +3 -3
- data/lib/karafka/pro/loader.rb +1 -1
- data/lib/karafka/pro/processing/coordinator.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
- data/lib/karafka/pro/processing/filters/base.rb +1 -0
- data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
- data/lib/karafka/pro/processing/partitioner.rb +1 -1
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/base.rb +1 -1
- data/lib/karafka/pro/processing/strategies/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
- data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +4 -6
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +3 -5
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +7 -6
- data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
- data/lib/karafka/pro/recurring_tasks/schedule.rb +9 -8
- data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
- data/lib/karafka/pro/recurring_tasks.rb +8 -5
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
- data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
- data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +4 -6
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +3 -5
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
- data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
- data/lib/karafka/pro/scheduled_messages.rb +4 -6
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +4 -4
- data/lib/karafka/processing/coordinator.rb +2 -4
- data/lib/karafka/processing/coordinators_buffer.rb +2 -3
- data/lib/karafka/processing/executor.rb +3 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
- data/lib/karafka/processing/jobs/base.rb +2 -3
- data/lib/karafka/processing/jobs_queue.rb +1 -1
- data/lib/karafka/processing/result.rb +1 -0
- data/lib/karafka/processing/strategy_selector.rb +1 -0
- data/lib/karafka/processing/workers_batch.rb +2 -3
- data/lib/karafka/railtie.rb +1 -0
- data/lib/karafka/routing/activity_manager.rb +3 -2
- data/lib/karafka/routing/builder.rb +8 -8
- data/lib/karafka/routing/consumer_group.rb +4 -6
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -7
- data/lib/karafka/routing/contracts/routing.rb +2 -4
- data/lib/karafka/routing/contracts/topic.rb +7 -6
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/active_job/topic.rb +6 -0
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights.rb +5 -5
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/subscription_groups_builder.rb +1 -0
- data/lib/karafka/routing/topic.rb +3 -3
- data/lib/karafka/routing/topics.rb +4 -9
- data/lib/karafka/server.rb +2 -2
- data/lib/karafka/setup/attributes_map.rb +4 -2
- data/lib/karafka/setup/config.rb +85 -17
- data/lib/karafka/setup/config_proxy.rb +209 -0
- data/lib/karafka/setup/contracts/config.rb +13 -11
- data/lib/karafka/setup/defaults_injector.rb +3 -2
- data/lib/karafka/setup/dsl.rb +2 -3
- data/lib/karafka/swarm/liveness_listener.rb +3 -3
- data/lib/karafka/swarm/manager.rb +7 -6
- data/lib/karafka/swarm/node.rb +1 -1
- data/lib/karafka/swarm/supervisor.rb +2 -1
- data/lib/karafka/time_trackers/base.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +4 -4
- metadata +14 -6
- data/.diffend.yml +0 -3
|
@@ -12,19 +12,15 @@ module Karafka
|
|
|
12
12
|
# Contract to validate configuration of the direct assignments feature
|
|
13
13
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
14
14
|
configure do |config|
|
|
15
|
-
config.error_messages = YAML.
|
|
16
|
-
File.
|
|
17
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
18
|
-
)
|
|
15
|
+
config.error_messages = YAML.safe_load_file(
|
|
16
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
17
|
).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
|
|
20
18
|
|
|
21
19
|
virtual do |data, errors|
|
|
22
20
|
next unless errors.empty?
|
|
23
21
|
|
|
24
|
-
active = []
|
|
25
|
-
|
|
26
|
-
data[:topics].each do |topic|
|
|
27
|
-
active << topic[:direct_assignments][:active]
|
|
22
|
+
active = data[:topics].map do |topic|
|
|
23
|
+
topic[:direct_assignments][:active]
|
|
28
24
|
end
|
|
29
25
|
|
|
30
26
|
# If none active we use standard subscriptions
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the direct assignments topic feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -26,8 +24,8 @@ module Karafka
|
|
|
26
24
|
required(:partitions) do |val|
|
|
27
25
|
next true if val == true
|
|
28
26
|
next false unless val.is_a?(Hash)
|
|
29
|
-
next false unless val.keys.all?
|
|
30
|
-
next false unless val.values.all?
|
|
27
|
+
next false unless val.keys.all?(Integer)
|
|
28
|
+
next false unless val.values.all?(true)
|
|
31
29
|
|
|
32
30
|
true
|
|
33
31
|
end
|
|
@@ -67,7 +65,7 @@ module Karafka
|
|
|
67
65
|
direct_partitions = direct_assignments[:partitions].keys
|
|
68
66
|
swarm_partitions = nodes.values.flatten
|
|
69
67
|
|
|
70
|
-
next unless swarm_partitions.all?
|
|
68
|
+
next unless swarm_partitions.all?(Integer)
|
|
71
69
|
next if direct_partitions.sort == swarm_partitions.sort
|
|
72
70
|
|
|
73
71
|
# If we assigned more partitions than we distributed in swarm
|
|
@@ -13,6 +13,10 @@ module Karafka
|
|
|
13
13
|
# Extension allowing us to select correct subscriptions and assignments based on the
|
|
14
14
|
# expanded routing setup
|
|
15
15
|
module SubscriptionGroup
|
|
16
|
+
include Helpers::ConfigImporter.new(
|
|
17
|
+
swarm_node: %i[swarm node]
|
|
18
|
+
)
|
|
19
|
+
|
|
16
20
|
# @return [false, Array<String>] false if we do not have any subscriptions or array
|
|
17
21
|
# with all the subscriptions for given subscription group
|
|
18
22
|
def subscriptions
|
|
@@ -30,8 +34,7 @@ module Karafka
|
|
|
30
34
|
topics
|
|
31
35
|
.select(&:active?)
|
|
32
36
|
.select { |topic| topic.direct_assignments.active? }
|
|
33
|
-
.
|
|
34
|
-
.to_h
|
|
37
|
+
.to_h { |topic| build_assignments(topic) }
|
|
35
38
|
.tap { |topics| return false if topics.empty? }
|
|
36
39
|
.then { |topics| Iterator::Expander.new.call(topics) }
|
|
37
40
|
.then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
|
|
@@ -44,21 +47,19 @@ module Karafka
|
|
|
44
47
|
# @param topic [Karafka::Routing::Topic]
|
|
45
48
|
# @return [Array<String, Hash>]
|
|
46
49
|
def build_assignments(topic)
|
|
47
|
-
node = Karafka::App.config.swarm.node
|
|
48
|
-
|
|
49
50
|
standard_setup = [
|
|
50
51
|
topic.subscription_name,
|
|
51
52
|
topic.direct_assignments.partitions
|
|
52
53
|
]
|
|
53
54
|
|
|
54
|
-
return standard_setup unless
|
|
55
|
+
return standard_setup unless swarm_node
|
|
55
56
|
# Unless user explicitly assigned particular partitions to particular nodes, we just
|
|
56
57
|
# go with full regular assignments
|
|
57
58
|
return standard_setup unless topic.swarm.nodes.is_a?(Hash)
|
|
58
59
|
|
|
59
60
|
[
|
|
60
61
|
topic.subscription_name,
|
|
61
|
-
topic.swarm.nodes.fetch(
|
|
62
|
+
topic.swarm.nodes.fetch(swarm_node.id).to_h { |partition| [partition, true] }
|
|
62
63
|
]
|
|
63
64
|
end
|
|
64
65
|
end
|
|
@@ -43,12 +43,12 @@ module Karafka
|
|
|
43
43
|
|
|
44
44
|
Config.new(
|
|
45
45
|
active: true,
|
|
46
|
-
partitions: partitions_or_all.
|
|
46
|
+
partitions: partitions_or_all.to_h { |partition| [partition, true] }
|
|
47
47
|
)
|
|
48
48
|
else
|
|
49
49
|
Config.new(
|
|
50
50
|
active: !partitions_or_all.empty?,
|
|
51
|
-
partitions: partitions_or_all.
|
|
51
|
+
partitions: partitions_or_all.to_h { |partition| [partition, true] }
|
|
52
52
|
)
|
|
53
53
|
end
|
|
54
54
|
end
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the expiring feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the filtering feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -27,10 +27,9 @@ module Karafka
|
|
|
27
27
|
@filtering
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
# @param args [Array] Anything `#filter` accepts
|
|
31
30
|
# @return [Filtering::Config] alias to match the naming API for features
|
|
32
|
-
def filtering(*
|
|
33
|
-
filter(*
|
|
31
|
+
def filtering(*)
|
|
32
|
+
filter(*)
|
|
34
33
|
end
|
|
35
34
|
|
|
36
35
|
# @return [Boolean] is a given job throttled
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract for inline insights topic setup
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around long-running job settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -17,10 +17,8 @@ module Karafka
|
|
|
17
17
|
# multiplexing attributes are optional since multiplexing may not be enabled
|
|
18
18
|
class Topic < Karafka::Contracts::Base
|
|
19
19
|
configure do |config|
|
|
20
|
-
config.error_messages = YAML.
|
|
21
|
-
File.
|
|
22
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
23
|
-
)
|
|
20
|
+
config.error_messages = YAML.safe_load_file(
|
|
21
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
24
22
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
25
23
|
end
|
|
26
24
|
|
|
@@ -56,7 +54,7 @@ module Karafka
|
|
|
56
54
|
max = max(data)
|
|
57
55
|
boot = boot(data)
|
|
58
56
|
|
|
59
|
-
next if boot
|
|
57
|
+
next if boot.between?(min, max)
|
|
60
58
|
|
|
61
59
|
[[%w[subscription_group_details], :multiplexing_boot_mismatch]]
|
|
62
60
|
end
|
|
@@ -19,7 +19,7 @@ module Karafka
|
|
|
19
19
|
factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
|
|
20
20
|
|
|
21
21
|
Array.new(factor) do |i|
|
|
22
|
-
|
|
22
|
+
Karafka::Routing::Topics.new(
|
|
23
23
|
i.zero? ? topics_array : topics_array.map(&:dup)
|
|
24
24
|
)
|
|
25
25
|
end
|
|
@@ -17,7 +17,7 @@ module Karafka
|
|
|
17
17
|
# @param _config [Karafka::Core::Configurable::Node] app config node
|
|
18
18
|
def pre_setup(_config)
|
|
19
19
|
# Make sure we use proper unique validator for topics definitions
|
|
20
|
-
|
|
20
|
+
Karafka::Routing::Contracts::ConsumerGroup.singleton_class.prepend(
|
|
21
21
|
Patches::Contracts::ConsumerGroup
|
|
22
22
|
)
|
|
23
23
|
end
|
|
@@ -26,11 +26,11 @@ module Karafka
|
|
|
26
26
|
#
|
|
27
27
|
# @param _config [Karafka::Core::Configurable::Node] app config
|
|
28
28
|
def post_setup(_config)
|
|
29
|
-
|
|
29
|
+
Karafka::App.monitor.subscribe('app.running') do
|
|
30
30
|
# Do not install the manager and listener to control multiplexing unless there is
|
|
31
31
|
# multiplexing enabled and it is dynamic.
|
|
32
32
|
# We only need to control multiplexing when it is in a dynamic state
|
|
33
|
-
next unless
|
|
33
|
+
next unless Karafka::App
|
|
34
34
|
.subscription_groups
|
|
35
35
|
.values
|
|
36
36
|
.flat_map(&:itself)
|
|
@@ -38,8 +38,8 @@ module Karafka
|
|
|
38
38
|
|
|
39
39
|
# Subscribe for events and possibility to manage via the Pro connection manager
|
|
40
40
|
# that supports multiplexing
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
Karafka.monitor.subscribe(
|
|
42
|
+
Karafka::Pro::Connection::Multiplexing::Listener.new
|
|
43
43
|
)
|
|
44
44
|
end
|
|
45
45
|
end
|
|
@@ -10,9 +10,9 @@ module Karafka
|
|
|
10
10
|
class NonBlockingJob < Base
|
|
11
11
|
# Non-Blocking Jobs topic API extensions
|
|
12
12
|
module Topic
|
|
13
|
-
#
|
|
14
|
-
def non_blocking_job(*
|
|
15
|
-
long_running_job(*
|
|
13
|
+
# Non blocking setting method
|
|
14
|
+
def non_blocking_job(*)
|
|
15
|
+
long_running_job(*)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
alias non_blocking non_blocking_job
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the expiring feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -14,13 +14,13 @@ module Karafka
|
|
|
14
14
|
#
|
|
15
15
|
# @param _config [Karafka::Core::Configurable::Node] app config
|
|
16
16
|
def post_setup(_config)
|
|
17
|
-
|
|
17
|
+
Karafka::App.monitor.subscribe('app.running') do
|
|
18
18
|
# Initialize the tracker prior to becoming multi-threaded
|
|
19
|
-
|
|
19
|
+
Karafka::Processing::InlineInsights::Tracker.instance
|
|
20
20
|
|
|
21
21
|
# Subscribe to the statistics reports and collect them
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
Karafka.monitor.subscribe(
|
|
23
|
+
Karafka::Pro::Processing::OffsetMetadata::Listener.new
|
|
24
24
|
)
|
|
25
25
|
end
|
|
26
26
|
end
|
|
@@ -23,7 +23,7 @@ module Karafka
|
|
|
23
23
|
# We build a temp consumer group and a target to check if it has parallel segments
|
|
24
24
|
# enabled and if so, we do not add it to the routing but instead we build the
|
|
25
25
|
# appropriate number of parallel segment groups
|
|
26
|
-
temp_consumer_group =
|
|
26
|
+
temp_consumer_group = Karafka::Routing::ConsumerGroup.new(group_id.to_s)
|
|
27
27
|
temp_target = Karafka::Routing::Proxy.new(temp_consumer_group, &block).target
|
|
28
28
|
config = temp_target.parallel_segments
|
|
29
29
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the parallel segments feature
|
|
14
14
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
|
|
21
19
|
|
|
22
20
|
nested(:parallel_segments) do
|
|
@@ -12,13 +12,11 @@ module Karafka
|
|
|
12
12
|
# Contract to validate configuration of the filtering feature
|
|
13
13
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
14
14
|
configure do |config|
|
|
15
|
-
config.error_messages = YAML.
|
|
16
|
-
File.
|
|
17
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
18
|
-
)
|
|
15
|
+
config.error_messages = YAML.safe_load_file(
|
|
16
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
17
|
).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
|
|
20
18
|
|
|
21
|
-
required(:patterns) { |val| val.is_a?(Array) && val.all?
|
|
19
|
+
required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
|
|
22
20
|
|
|
23
21
|
virtual do |data, errors|
|
|
24
22
|
next unless errors.empty?
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract used to validate pattern data
|
|
14
14
|
class Pattern < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('pattern')
|
|
21
19
|
|
|
22
20
|
required(:regexp) { |val| val.is_a?(Regexp) }
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the patterns feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -9,7 +9,7 @@ module Karafka
|
|
|
9
9
|
module Features
|
|
10
10
|
class Patterns < Base
|
|
11
11
|
# Representation of groups of topics
|
|
12
|
-
class Patterns <
|
|
12
|
+
class Patterns < Karafka::Routing::Topics
|
|
13
13
|
# Finds first pattern matching given topic name
|
|
14
14
|
#
|
|
15
15
|
# @param topic_name [String] topic name that may match a pattern
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
|
|
4
|
+
# See LICENSE for details.
|
|
5
|
+
|
|
6
|
+
module Karafka
|
|
7
|
+
module Pro
|
|
8
|
+
module Routing
|
|
9
|
+
module Features
|
|
10
|
+
class Pausing < Base
|
|
11
|
+
# Config for pausing feature
|
|
12
|
+
Config = Struct.new(
|
|
13
|
+
:active,
|
|
14
|
+
:timeout,
|
|
15
|
+
:max_timeout,
|
|
16
|
+
:with_exponential_backoff,
|
|
17
|
+
keyword_init: true
|
|
18
|
+
) do
|
|
19
|
+
alias_method :active?, :active
|
|
20
|
+
alias_method :with_exponential_backoff?, :with_exponential_backoff
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -13,25 +13,31 @@ module Karafka
|
|
|
13
13
|
# Contract to make sure, that the pause settings on a per topic basis are as expected
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
# Validate the nested pausing configuration
|
|
21
|
+
# Both old setters and new pause() method update the pausing config,
|
|
22
|
+
# so we only need to validate this one format
|
|
23
|
+
nested :pausing do
|
|
24
|
+
required(:active) { |val| [true, false].include?(val) }
|
|
25
|
+
required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
26
|
+
required(:max_timeout) { |val| val.is_a?(Integer) && val.positive? }
|
|
27
|
+
required(:with_exponential_backoff) { |val| [true, false].include?(val) }
|
|
28
|
+
end
|
|
25
29
|
|
|
30
|
+
# Validate that timeout <= max_timeout
|
|
26
31
|
virtual do |data, errors|
|
|
27
32
|
next unless errors.empty?
|
|
28
33
|
|
|
29
|
-
|
|
30
|
-
|
|
34
|
+
pausing = data.fetch(:pausing)
|
|
35
|
+
timeout = pausing.fetch(:timeout)
|
|
36
|
+
max_timeout = pausing.fetch(:max_timeout)
|
|
31
37
|
|
|
32
|
-
next if
|
|
38
|
+
next if timeout <= max_timeout
|
|
33
39
|
|
|
34
|
-
[[%i[
|
|
40
|
+
[[%i[pausing timeout], :max_timeout_vs_pause_max_timeout]]
|
|
35
41
|
end
|
|
36
42
|
end
|
|
37
43
|
end
|
|
@@ -10,23 +10,84 @@ module Karafka
|
|
|
10
10
|
class Pausing < Base
|
|
11
11
|
# Expansion allowing for a per topic pause strategy definitions
|
|
12
12
|
module Topic
|
|
13
|
+
# This method calls the parent class initializer and then sets up the
|
|
14
|
+
# extra instance variable to nil. The explicit initialization
|
|
15
|
+
# to nil is included as an optimization for Ruby's object shapes system,
|
|
16
|
+
# which improves memory layout and access performance.
|
|
17
|
+
def initialize(...)
|
|
18
|
+
super
|
|
19
|
+
@pausing = nil
|
|
20
|
+
end
|
|
21
|
+
|
|
13
22
|
# Allows for per-topic pausing strategy setting
|
|
14
23
|
#
|
|
15
24
|
# @param timeout [Integer] how long should we wait upon processing error (milliseconds)
|
|
16
25
|
# @param max_timeout [Integer] what is the max timeout in case of an exponential
|
|
17
26
|
# backoff (milliseconds)
|
|
18
27
|
# @param with_exponential_backoff [Boolean] should we use exponential backoff
|
|
19
|
-
#
|
|
20
|
-
# @note We do not construct here the nested config like we do with other routing
|
|
21
|
-
# features, because this feature operates on the OSS layer by injection of values
|
|
22
|
-
# and a nested config is not needed.
|
|
28
|
+
# @return [Config] pausing config object
|
|
23
29
|
def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
|
|
24
|
-
|
|
25
|
-
|
|
30
|
+
# If no arguments provided, just return or initialize the config
|
|
31
|
+
return pausing if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
|
|
32
|
+
|
|
33
|
+
# Update instance variables for backwards compatibility
|
|
34
|
+
# This ensures code reading @pause_timeout directly or via the inherited getter
|
|
35
|
+
# will get the correct values
|
|
36
|
+
@pause_timeout = timeout if timeout
|
|
37
|
+
@pause_max_timeout = max_timeout if max_timeout
|
|
38
|
+
|
|
39
|
+
unless with_exponential_backoff.nil?
|
|
40
|
+
@pause_with_exponential_backoff = with_exponential_backoff
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Create or update the config
|
|
44
|
+
@pausing ||= Config.new(
|
|
45
|
+
active: false,
|
|
46
|
+
timeout: @pause_timeout || Karafka::App.config.pause.timeout,
|
|
47
|
+
max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
|
|
48
|
+
with_exponential_backoff: if @pause_with_exponential_backoff.nil?
|
|
49
|
+
Karafka::App.config.pause.with_exponential_backoff
|
|
50
|
+
else
|
|
51
|
+
@pause_with_exponential_backoff
|
|
52
|
+
end
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
@pausing.timeout = timeout if timeout
|
|
56
|
+
@pausing.max_timeout = max_timeout if max_timeout
|
|
26
57
|
|
|
27
|
-
|
|
58
|
+
unless with_exponential_backoff.nil?
|
|
59
|
+
@pausing.with_exponential_backoff = with_exponential_backoff
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
@pausing.active = true
|
|
63
|
+
|
|
64
|
+
@pausing
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# @return [Config] pausing configuration object
|
|
68
|
+
def pausing
|
|
69
|
+
@pausing ||= Config.new(
|
|
70
|
+
active: false,
|
|
71
|
+
timeout: @pause_timeout || Karafka::App.config.pause.timeout,
|
|
72
|
+
max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
|
|
73
|
+
with_exponential_backoff: if @pause_with_exponential_backoff.nil?
|
|
74
|
+
Karafka::App.config.pause.with_exponential_backoff
|
|
75
|
+
else
|
|
76
|
+
@pause_with_exponential_backoff
|
|
77
|
+
end
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# @return [Boolean] is pausing explicitly configured
|
|
82
|
+
def pausing?
|
|
83
|
+
pausing.active?
|
|
84
|
+
end
|
|
28
85
|
|
|
29
|
-
|
|
86
|
+
# @return [Hash] topic with all its native configuration options plus pausing settings
|
|
87
|
+
def to_h
|
|
88
|
+
super.merge(
|
|
89
|
+
pausing: pausing.to_h
|
|
90
|
+
).freeze
|
|
30
91
|
end
|
|
31
92
|
end
|
|
32
93
|
end
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the periodics feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -108,7 +108,7 @@ module Karafka
|
|
|
108
108
|
require 'fugit'
|
|
109
109
|
rescue LoadError
|
|
110
110
|
raise(
|
|
111
|
-
|
|
111
|
+
Karafka::Errors::DependencyConstraintsError,
|
|
112
112
|
<<~ERROR_MSG
|
|
113
113
|
Failed to require fugit gem.
|
|
114
114
|
Add it to your Gemfile, as it is required for the recurring tasks to work.
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around recurring tasks settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around scheduled messages settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|