karafka 2.5.1 → 2.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci_linux_ubuntu_x86_64_gnu.yml +21 -29
- data/.github/workflows/ci_macos_arm64.yml +1 -1
- data/.github/workflows/push.yml +2 -2
- data/.github/workflows/trigger-wiki-refresh.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +174 -0
- data/CHANGELOG.md +20 -4
- data/Gemfile +1 -2
- data/Gemfile.lock +45 -41
- data/bin/integrations +2 -1
- data/bin/rspecs +4 -0
- data/config/locales/errors.yml +6 -4
- data/config/locales/pro_errors.yml +5 -4
- data/docker-compose.yml +1 -1
- data/examples/payloads/json/sample_set_02/download.json +191 -0
- data/examples/payloads/json/sample_set_03/event_type_1.json +18 -0
- data/examples/payloads/json/sample_set_03/event_type_2.json +263 -0
- data/examples/payloads/json/sample_set_03/event_type_3.json +41 -0
- data/karafka.gemspec +3 -3
- data/lib/active_job/queue_adapters/karafka_adapter.rb +3 -3
- data/lib/karafka/active_job/consumer.rb +7 -3
- data/lib/karafka/active_job/current_attributes/job_wrapper.rb +45 -0
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes/persistence.rb +19 -7
- data/lib/karafka/active_job/current_attributes.rb +3 -2
- data/lib/karafka/active_job/deserializer.rb +61 -0
- data/lib/karafka/active_job/dispatcher.rb +34 -14
- data/lib/karafka/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/admin/acl.rb +8 -4
- data/lib/karafka/admin/configs/config.rb +6 -4
- data/lib/karafka/admin/configs/resource.rb +7 -1
- data/lib/karafka/admin/consumer_groups.rb +80 -12
- data/lib/karafka/admin/topics.rb +43 -9
- data/lib/karafka/admin.rb +23 -14
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +6 -6
- data/lib/karafka/cli/base.rb +2 -2
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +3 -5
- data/lib/karafka/cli/help.rb +1 -1
- data/lib/karafka/cli/install.rb +3 -2
- data/lib/karafka/cli/server.rb +1 -1
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +1 -1
- data/lib/karafka/cli/topics/repartition.rb +2 -2
- data/lib/karafka/connection/client.rb +30 -19
- data/lib/karafka/connection/listeners_batch.rb +2 -3
- data/lib/karafka/connection/manager.rb +1 -0
- data/lib/karafka/connection/proxy.rb +12 -8
- data/lib/karafka/connection/rebalance_manager.rb +1 -1
- data/lib/karafka/connection/status.rb +1 -0
- data/lib/karafka/constraints.rb +1 -1
- data/lib/karafka/contracts/base.rb +1 -1
- data/lib/karafka/deserializers/payload.rb +1 -1
- data/lib/karafka/env.rb +1 -2
- data/lib/karafka/helpers/async.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +3 -3
- data/lib/karafka/helpers/interval_runner.rb +4 -1
- data/lib/karafka/helpers/multi_delegator.rb +3 -0
- data/lib/karafka/instrumentation/assignments_tracker.rb +19 -1
- data/lib/karafka/instrumentation/callbacks/error.rb +2 -2
- data/lib/karafka/instrumentation/callbacks/statistics.rb +3 -3
- data/lib/karafka/instrumentation/logger.rb +6 -6
- data/lib/karafka/instrumentation/monitor.rb +3 -3
- data/lib/karafka/instrumentation/notifications.rb +1 -0
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +3 -4
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +3 -4
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +10 -11
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +5 -18
- data/lib/karafka/messages/builders/batch_metadata.rb +2 -2
- data/lib/karafka/messages/builders/message.rb +1 -1
- data/lib/karafka/messages/messages.rb +2 -3
- data/lib/karafka/patches/rdkafka/bindings.rb +6 -6
- data/lib/karafka/patches/rdkafka/opaque.rb +1 -1
- data/lib/karafka/pro/active_job/consumer.rb +2 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +10 -6
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -4
- data/lib/karafka/pro/cleaner/messages/messages.rb +2 -3
- data/lib/karafka/pro/cleaner.rb +3 -3
- data/lib/karafka/pro/cli/contracts/server.rb +3 -5
- data/lib/karafka/pro/cli/parallel_segments/base.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +1 -1
- data/lib/karafka/pro/connection/manager.rb +3 -4
- data/lib/karafka/pro/connection/multiplexing/listener.rb +1 -0
- data/lib/karafka/pro/contracts/base.rb +1 -1
- data/lib/karafka/pro/encryption/cipher.rb +3 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +5 -7
- data/lib/karafka/pro/encryption/messages/parser.rb +4 -4
- data/lib/karafka/pro/encryption/setup/config.rb +1 -1
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +3 -3
- data/lib/karafka/pro/iterator/expander.rb +1 -1
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +3 -3
- data/lib/karafka/pro/loader.rb +1 -1
- data/lib/karafka/pro/processing/coordinator.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -3
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +3 -3
- data/lib/karafka/pro/processing/filters/base.rb +1 -0
- data/lib/karafka/pro/processing/filters/delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/expirer.rb +1 -1
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/filters/throttler.rb +1 -1
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs/periodic.rb +1 -1
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +1 -1
- data/lib/karafka/pro/processing/jobs_builder.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +1 -0
- data/lib/karafka/pro/processing/partitioner.rb +1 -1
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/base.rb +1 -1
- data/lib/karafka/pro/processing/strategies/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -4
- data/lib/karafka/pro/processing/strategies/vp/default.rb +2 -4
- data/lib/karafka/pro/processing/strategy_selector.rb +1 -0
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -3
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +4 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +4 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +3 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +4 -6
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +3 -5
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +7 -6
- data/lib/karafka/pro/recurring_tasks/executor.rb +2 -1
- data/lib/karafka/pro/recurring_tasks/schedule.rb +9 -8
- data/lib/karafka/pro/recurring_tasks/serializer.rb +6 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/task.rb +1 -1
- data/lib/karafka/pro/recurring_tasks.rb +8 -5
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +3 -0
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +4 -8
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +5 -7
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +7 -6
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -3
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +3 -5
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +3 -3
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/offset_metadata.rb +4 -4
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +3 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/config.rb +26 -0
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +17 -11
- data/lib/karafka/pro/routing/features/pausing/topic.rb +69 -8
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +6 -8
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +3 -1
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -4
- data/lib/karafka/pro/scheduled_messages/consumer.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +4 -6
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +3 -5
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +3 -2
- data/lib/karafka/pro/scheduled_messages/day.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/proxy.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/serializer.rb +3 -3
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -0
- data/lib/karafka/pro/scheduled_messages/tracker.rb +1 -0
- data/lib/karafka/pro/scheduled_messages.rb +4 -6
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +4 -4
- data/lib/karafka/processing/coordinator.rb +2 -4
- data/lib/karafka/processing/coordinators_buffer.rb +2 -3
- data/lib/karafka/processing/executor.rb +3 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +1 -0
- data/lib/karafka/processing/jobs/base.rb +2 -3
- data/lib/karafka/processing/jobs_queue.rb +1 -1
- data/lib/karafka/processing/result.rb +1 -0
- data/lib/karafka/processing/strategy_selector.rb +1 -0
- data/lib/karafka/processing/workers_batch.rb +2 -3
- data/lib/karafka/railtie.rb +1 -0
- data/lib/karafka/routing/activity_manager.rb +3 -2
- data/lib/karafka/routing/builder.rb +8 -8
- data/lib/karafka/routing/consumer_group.rb +4 -6
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -7
- data/lib/karafka/routing/contracts/routing.rb +2 -4
- data/lib/karafka/routing/contracts/topic.rb +7 -6
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/active_job/topic.rb +6 -0
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +3 -5
- data/lib/karafka/routing/features/declaratives/topic.rb +5 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -4
- data/lib/karafka/routing/features/inline_insights.rb +5 -5
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -4
- data/lib/karafka/routing/router.rb +1 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/subscription_groups_builder.rb +1 -0
- data/lib/karafka/routing/topic.rb +3 -3
- data/lib/karafka/routing/topics.rb +4 -9
- data/lib/karafka/server.rb +2 -2
- data/lib/karafka/setup/attributes_map.rb +4 -2
- data/lib/karafka/setup/config.rb +85 -17
- data/lib/karafka/setup/config_proxy.rb +209 -0
- data/lib/karafka/setup/contracts/config.rb +13 -11
- data/lib/karafka/setup/defaults_injector.rb +3 -2
- data/lib/karafka/setup/dsl.rb +2 -3
- data/lib/karafka/swarm/liveness_listener.rb +3 -3
- data/lib/karafka/swarm/manager.rb +7 -6
- data/lib/karafka/swarm/node.rb +1 -1
- data/lib/karafka/swarm/supervisor.rb +2 -1
- data/lib/karafka/time_trackers/base.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +4 -4
- metadata +14 -6
- data/.diffend.yml +0 -3
|
@@ -18,10 +18,8 @@ module Karafka
|
|
|
18
18
|
# This is why we use it before warmup when everything is expected to be configured.
|
|
19
19
|
class Routing < Karafka::Contracts::Base
|
|
20
20
|
configure do |config|
|
|
21
|
-
config.error_messages = YAML.
|
|
22
|
-
File.
|
|
23
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
24
|
-
)
|
|
21
|
+
config.error_messages = YAML.safe_load_file(
|
|
22
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
25
23
|
).fetch('en').fetch('validations').fetch('routing')
|
|
26
24
|
end
|
|
27
25
|
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Contract to validate configuration of the swarm feature
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -25,15 +23,15 @@ module Karafka
|
|
|
25
23
|
|
|
26
24
|
required(:nodes) do |val|
|
|
27
25
|
next true if val.is_a?(Range)
|
|
28
|
-
next true if val.is_a?(Array) && val.all?
|
|
26
|
+
next true if val.is_a?(Array) && val.all?(Integer)
|
|
29
27
|
next false unless val.is_a?(Hash)
|
|
30
|
-
next false unless val.keys.all?
|
|
28
|
+
next false unless val.keys.all?(Integer)
|
|
31
29
|
|
|
32
30
|
values = val.values
|
|
33
31
|
|
|
34
32
|
next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
|
|
35
|
-
next true if values.flatten.all?
|
|
36
|
-
next true if values.flatten.all?
|
|
33
|
+
next true if values.flatten.all?(Integer)
|
|
34
|
+
next true if values.flatten.all?(Range)
|
|
37
35
|
|
|
38
36
|
false
|
|
39
37
|
end
|
|
@@ -14,7 +14,7 @@ module Karafka
|
|
|
14
14
|
# Binds our routing validation contract prior to warmup in the supervisor, so we can
|
|
15
15
|
# run it when all the context should be there (config + full routing)
|
|
16
16
|
#
|
|
17
|
-
# @param config [Karafka::Core::Configurable::Node]
|
|
17
|
+
# @param config [Karafka::Core::Configurable::Node]
|
|
18
18
|
def post_setup(config)
|
|
19
19
|
config.monitor.subscribe('app.before_warmup') do
|
|
20
20
|
Contracts::Routing.new.validate!(
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around throttling settings
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -46,7 +46,9 @@ module Karafka
|
|
|
46
46
|
|
|
47
47
|
# Just an alias for nice API
|
|
48
48
|
#
|
|
49
|
-
# @param args [
|
|
49
|
+
# @param args [Hash] Anything `#throttling` accepts
|
|
50
|
+
# @option args [Integer] :limit max messages to process in a time interval
|
|
51
|
+
# @option args [Integer] :interval time interval for processing in milliseconds
|
|
50
52
|
def throttle(**args)
|
|
51
53
|
throttling(**args)
|
|
52
54
|
end
|
|
@@ -13,10 +13,8 @@ module Karafka
|
|
|
13
13
|
# Rules around virtual partitions
|
|
14
14
|
class Topic < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('routing').fetch('topic')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -7,7 +7,7 @@ module Karafka
|
|
|
7
7
|
module Pro
|
|
8
8
|
module ScheduledMessages
|
|
9
9
|
# Consumer that coordinates scheduling of messages when the time comes
|
|
10
|
-
class Consumer <
|
|
10
|
+
class Consumer < Karafka::BaseConsumer
|
|
11
11
|
include Helpers::ConfigImporter.new(
|
|
12
12
|
dispatcher_class: %i[scheduled_messages dispatcher_class]
|
|
13
13
|
)
|
|
@@ -9,17 +9,15 @@ module Karafka
|
|
|
9
9
|
# Recurring Tasks related contracts
|
|
10
10
|
module Contracts
|
|
11
11
|
# Makes sure, all the expected config is defined as it should be
|
|
12
|
-
class Config <
|
|
12
|
+
class Config < Karafka::Contracts::Base
|
|
13
13
|
configure do |config|
|
|
14
|
-
config.error_messages = YAML.
|
|
15
|
-
File.
|
|
16
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
17
|
-
)
|
|
14
|
+
config.error_messages = YAML.safe_load_file(
|
|
15
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
18
16
|
).fetch('en').fetch('validations').fetch('setup').fetch('config')
|
|
19
17
|
end
|
|
20
18
|
|
|
21
19
|
nested(:scheduled_messages) do
|
|
22
|
-
required(:consumer_class) { |val| val <
|
|
20
|
+
required(:consumer_class) { |val| val < Karafka::BaseConsumer }
|
|
23
21
|
|
|
24
22
|
# Do not allow to run more often than every second
|
|
25
23
|
required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
|
|
@@ -11,12 +11,10 @@ module Karafka
|
|
|
11
11
|
#
|
|
12
12
|
# Our envelope always needs to comply with this format, otherwise we won't have enough
|
|
13
13
|
# details to be able to dispatch the message
|
|
14
|
-
class Message <
|
|
14
|
+
class Message < Karafka::Contracts::Base
|
|
15
15
|
configure do |config|
|
|
16
|
-
config.error_messages = YAML.
|
|
17
|
-
File.
|
|
18
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
19
|
-
)
|
|
16
|
+
config.error_messages = YAML.safe_load_file(
|
|
17
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
|
|
20
18
|
).fetch('en').fetch('validations').fetch('scheduled_messages_message')
|
|
21
19
|
end
|
|
22
20
|
|
|
@@ -14,6 +14,7 @@ module Karafka
|
|
|
14
14
|
# up to 1.5 second, thus it is acceptable. Please ping me if you encounter performance
|
|
15
15
|
# issues with this naive implementation so it can be improved.
|
|
16
16
|
class DailyBuffer
|
|
17
|
+
# Initializes the daily buffer with empty accumulator
|
|
17
18
|
def initialize
|
|
18
19
|
@accu = {}
|
|
19
20
|
end
|
|
@@ -69,8 +70,8 @@ module Karafka
|
|
|
69
70
|
selected.each { |_, message| yield(message) }
|
|
70
71
|
end
|
|
71
72
|
|
|
72
|
-
# Removes given key from the
|
|
73
|
-
# @param key [String]
|
|
73
|
+
# Removes the schedule entry identified by the given key from the daily buffer
|
|
74
|
+
# @param key [String]
|
|
74
75
|
def delete(key)
|
|
75
76
|
@accu.delete(key)
|
|
76
77
|
end
|
|
@@ -21,7 +21,7 @@ module Karafka
|
|
|
21
21
|
module Proxy
|
|
22
22
|
# General WaterDrop message contract. Before we envelop a message, we need to be certain
|
|
23
23
|
# it is correct, hence we use this contract.
|
|
24
|
-
MSG_CONTRACT =
|
|
24
|
+
MSG_CONTRACT = WaterDrop::Contracts::Message.new(
|
|
25
25
|
# Payload size is a subject to the target producer dispatch validation, so we set it
|
|
26
26
|
# to 100MB basically to ignore it here.
|
|
27
27
|
max_payload_size: 104_857_600
|
|
@@ -9,7 +9,7 @@ module Karafka
|
|
|
9
9
|
# Serializers used to build payloads (if applicable) for dispatch
|
|
10
10
|
# @note We only deal with states payload. Other payloads are not ours but end users.
|
|
11
11
|
class Serializer
|
|
12
|
-
include
|
|
12
|
+
include Karafka::Core::Helpers::Time
|
|
13
13
|
|
|
14
14
|
# @param tracker [Tracker] tracker based on which we build the state
|
|
15
15
|
# @return [String] compressed payload with the state details
|
|
@@ -32,9 +32,9 @@ module Karafka
|
|
|
32
32
|
hash.to_json
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
# Compresses the provided data
|
|
35
|
+
# Compresses the provided data using Zlib deflate algorithm
|
|
36
36
|
#
|
|
37
|
-
# @param data [String]
|
|
37
|
+
# @param data [String]
|
|
38
38
|
# @return [String] compressed data
|
|
39
39
|
def compress(data)
|
|
40
40
|
Zlib::Deflate.deflate(data)
|
|
@@ -10,7 +10,7 @@ module Karafka
|
|
|
10
10
|
module Setup
|
|
11
11
|
# Config for recurring tasks
|
|
12
12
|
class Config
|
|
13
|
-
extend
|
|
13
|
+
extend Karafka::Core::Configurable
|
|
14
14
|
|
|
15
15
|
setting(:consumer_class, default: Consumer)
|
|
16
16
|
setting(:group_id, default: 'karafka_scheduled_messages')
|
|
@@ -27,7 +27,7 @@ module Karafka
|
|
|
27
27
|
# Producer to use. By default uses default Karafka producer.
|
|
28
28
|
setting(
|
|
29
29
|
:producer,
|
|
30
|
-
constructor: -> {
|
|
30
|
+
constructor: -> { Karafka.producer },
|
|
31
31
|
lazy: true
|
|
32
32
|
)
|
|
33
33
|
|
|
@@ -24,17 +24,15 @@ module Karafka
|
|
|
24
24
|
|
|
25
25
|
class << self
|
|
26
26
|
# Runs the `Proxy.call`
|
|
27
|
-
# @param kwargs [Hash] things requested by the proxy
|
|
28
27
|
# @return [Hash] message wrapped with the scheduled message envelope
|
|
29
|
-
def schedule(**
|
|
30
|
-
Proxy.schedule(**
|
|
28
|
+
def schedule(**)
|
|
29
|
+
Proxy.schedule(**)
|
|
31
30
|
end
|
|
32
31
|
|
|
33
32
|
# Generates a tombstone message to cancel given dispatch (if not yet happened)
|
|
34
|
-
# @param kwargs [Hash] things requested by the proxy
|
|
35
33
|
# @return [Hash] tombstone cancelling message
|
|
36
|
-
def cancel(**
|
|
37
|
-
Proxy.cancel(**
|
|
34
|
+
def cancel(**)
|
|
35
|
+
Proxy.cancel(**)
|
|
38
36
|
end
|
|
39
37
|
|
|
40
38
|
# Below are private APIs
|
|
@@ -171,14 +171,14 @@ module Karafka
|
|
|
171
171
|
def rss_mb_linux
|
|
172
172
|
kb_rss = 0
|
|
173
173
|
|
|
174
|
-
|
|
174
|
+
File.readlines("/proc/#{node.pid}/status").each do |line|
|
|
175
175
|
next unless line.start_with?('VmRSS:')
|
|
176
176
|
|
|
177
177
|
kb_rss = line.split[1].to_i
|
|
178
178
|
break
|
|
179
179
|
end
|
|
180
180
|
|
|
181
|
-
(kb_rss / 1_024
|
|
181
|
+
(kb_rss / 1_024).round
|
|
182
182
|
rescue Errno::ENOENT, Errno::EACCES
|
|
183
183
|
# /proc file doesn't exist or no permission to read it
|
|
184
184
|
0
|
data/lib/karafka/process.rb
CHANGED
|
@@ -5,7 +5,7 @@ module Karafka
|
|
|
5
5
|
# @note There might be only one process - this class is a singleton
|
|
6
6
|
class Process
|
|
7
7
|
# Allow for process tagging for instrumentation
|
|
8
|
-
extend
|
|
8
|
+
extend Karafka::Core::Taggable
|
|
9
9
|
|
|
10
10
|
# Signal types that we handle
|
|
11
11
|
HANDLED_SIGNALS = %i[
|
|
@@ -79,12 +79,12 @@ module Karafka
|
|
|
79
79
|
private
|
|
80
80
|
|
|
81
81
|
# Traps a single signal and performs callbacks (if any) or just ignores this signal
|
|
82
|
-
# @param [Symbol] signal type that we want to catch
|
|
82
|
+
# @param signal [Symbol] signal type that we want to catch
|
|
83
83
|
# @note Since we do a lot of threading and queuing, we don't want to handle signals from the
|
|
84
84
|
# trap context s some things may not work there as expected, that is why we spawn a separate
|
|
85
85
|
# thread to handle the signals process
|
|
86
86
|
def trap_signal(signal)
|
|
87
|
-
previous_handler =
|
|
87
|
+
previous_handler = Signal.trap(signal) do
|
|
88
88
|
Thread.new do
|
|
89
89
|
notice_signal(signal)
|
|
90
90
|
|
|
@@ -96,7 +96,7 @@ module Karafka
|
|
|
96
96
|
end
|
|
97
97
|
|
|
98
98
|
# Informs monitoring about trapped signal
|
|
99
|
-
# @param [Symbol] signal type that we received
|
|
99
|
+
# @param signal [Symbol] signal type that we received
|
|
100
100
|
def notice_signal(signal)
|
|
101
101
|
Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
|
|
102
102
|
end
|
|
@@ -198,8 +198,6 @@ module Karafka
|
|
|
198
198
|
|
|
199
199
|
# Allows to run synchronized (locked) code that can operate only from a given thread
|
|
200
200
|
#
|
|
201
|
-
# @param block [Proc] code we want to run in the synchronized mode
|
|
202
|
-
#
|
|
203
201
|
# @note We check if mutex is not owned already by the current thread so we won't end up with
|
|
204
202
|
# a deadlock in case user runs coordinated code from inside of his own lock
|
|
205
203
|
#
|
|
@@ -207,11 +205,11 @@ module Karafka
|
|
|
207
205
|
# Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
|
|
208
206
|
# This can and should however be used for multi-thread strategy applications and other
|
|
209
207
|
# internal operations locks.
|
|
210
|
-
def synchronize(&
|
|
208
|
+
def synchronize(&)
|
|
211
209
|
if @mutex.owned?
|
|
212
210
|
yield
|
|
213
211
|
else
|
|
214
|
-
@mutex.synchronize(&
|
|
212
|
+
@mutex.synchronize(&)
|
|
215
213
|
end
|
|
216
214
|
end
|
|
217
215
|
end
|
|
@@ -36,11 +36,10 @@ module Karafka
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# Resumes processing of partitions for which pause time has ended.
|
|
39
|
-
# @param block we want to run for resumed topic partitions
|
|
40
39
|
# @yieldparam [String] topic name
|
|
41
40
|
# @yieldparam [Integer] partition number
|
|
42
|
-
def resume(&
|
|
43
|
-
@pauses_manager.resume(&
|
|
41
|
+
def resume(&)
|
|
42
|
+
@pauses_manager.resume(&)
|
|
44
43
|
end
|
|
45
44
|
|
|
46
45
|
# @param topic_name [String] topic name
|
|
@@ -83,9 +83,8 @@ module Karafka
|
|
|
83
83
|
|
|
84
84
|
# Runs the wrap/around execution context appropriate for a given action
|
|
85
85
|
# @param action [Symbol] action execution wrapped with our block
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
consumer.on_wrap(action, &block)
|
|
86
|
+
def wrap(action, &)
|
|
87
|
+
consumer.on_wrap(action, &)
|
|
89
88
|
end
|
|
90
89
|
|
|
91
90
|
# Runs consumer data processing against given batch and handles failures and errors.
|
|
@@ -185,7 +184,7 @@ module Karafka
|
|
|
185
184
|
# We assign producer only when not available already. It may already be available if
|
|
186
185
|
# user redefined the `#producer` method for example. This can be useful for example when
|
|
187
186
|
# having a multi-cluster setup and using a totally custom producer
|
|
188
|
-
consumer.producer ||=
|
|
187
|
+
consumer.producer ||= Karafka::App.producer
|
|
189
188
|
# Since we have some message-less flows (idle, etc), we initialize consumer with empty
|
|
190
189
|
# messages set. In production we have persistent consumers, so this is not a performance
|
|
191
190
|
# overhead as this will happen only once per consumer lifetime
|
|
@@ -29,11 +29,10 @@ module Karafka
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
# Runs the wrap/around job hook within which the rest of the flow happens
|
|
32
|
-
# @param block [Proc] whole user related processing flow
|
|
33
32
|
# @note We inject the action name so user can decide whether to run custom logic on a
|
|
34
33
|
# given action or not.
|
|
35
|
-
def wrap(&
|
|
36
|
-
executor.wrap(self.class.action, &
|
|
34
|
+
def wrap(&)
|
|
35
|
+
executor.wrap(self.class.action, &)
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
# When redefined can run any code prior to the job being scheduled
|
|
@@ -99,7 +99,7 @@ module Karafka
|
|
|
99
99
|
# Marks a given job from a given group as completed. When there are no more jobs from a given
|
|
100
100
|
# group to be executed, we won't wait.
|
|
101
101
|
#
|
|
102
|
-
# @param [Jobs::Base] job that was completed
|
|
102
|
+
# @param job [Jobs::Base] job that was completed
|
|
103
103
|
def complete(job)
|
|
104
104
|
@mutex.synchronize do
|
|
105
105
|
# We finish one job and if there is another, we pick it up
|
|
@@ -16,9 +16,8 @@ module Karafka
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
# Iterates over available workers and yields each worker
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@batch.each(&block)
|
|
19
|
+
def each(&)
|
|
20
|
+
@batch.each(&)
|
|
22
21
|
end
|
|
23
22
|
|
|
24
23
|
# @return [Integer] number of workers in the batch
|
data/lib/karafka/railtie.rb
CHANGED
|
@@ -49,6 +49,7 @@ if Karafka.rails?
|
|
|
49
49
|
rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
|
|
50
50
|
|
|
51
51
|
# Rails 7.1 replaced the broadcast module with a broadcast logger
|
|
52
|
+
# While 7.1 is EOL, we keep this for users who may still use it without official support
|
|
52
53
|
if rails71plus
|
|
53
54
|
Rails.logger.broadcast_to(stdout_logger)
|
|
54
55
|
else
|
|
@@ -12,6 +12,7 @@ module Karafka
|
|
|
12
12
|
topics
|
|
13
13
|
].freeze
|
|
14
14
|
|
|
15
|
+
# Initializes the activity manager with empty inclusion and exclusion lists
|
|
15
16
|
def initialize
|
|
16
17
|
@included = Hash.new { |h, k| h[k] = [] }
|
|
17
18
|
@excluded = Hash.new { |h, k| h[k] = [] }
|
|
@@ -58,8 +59,8 @@ module Karafka
|
|
|
58
59
|
# @return [Hash] accumulated data in a hash for validations
|
|
59
60
|
def to_h
|
|
60
61
|
(
|
|
61
|
-
SUPPORTED_TYPES.map { |type| ["include_#{type}"
|
|
62
|
-
SUPPORTED_TYPES.map { |type| ["exclude_#{type}"
|
|
62
|
+
SUPPORTED_TYPES.map { |type| [:"include_#{type}", @included[type]] } +
|
|
63
|
+
SUPPORTED_TYPES.map { |type| [:"exclude_#{type}", @excluded[type]] }
|
|
63
64
|
).to_h
|
|
64
65
|
end
|
|
65
66
|
|
|
@@ -23,6 +23,7 @@ module Karafka
|
|
|
23
23
|
|
|
24
24
|
private_constant :EMPTY_DEFAULTS
|
|
25
25
|
|
|
26
|
+
# Initializes the routing builder with empty routes
|
|
26
27
|
def initialize
|
|
27
28
|
@mutex = Mutex.new
|
|
28
29
|
@draws = []
|
|
@@ -81,13 +82,12 @@ module Karafka
|
|
|
81
82
|
private :array_clear
|
|
82
83
|
|
|
83
84
|
# Clear routes and draw them again with the given block. Helpful for testing purposes.
|
|
84
|
-
|
|
85
|
-
def redraw(&block)
|
|
85
|
+
def redraw(&)
|
|
86
86
|
@mutex.synchronize do
|
|
87
87
|
@draws.clear
|
|
88
88
|
array_clear
|
|
89
89
|
end
|
|
90
|
-
draw(&
|
|
90
|
+
draw(&)
|
|
91
91
|
end
|
|
92
92
|
|
|
93
93
|
# @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
|
|
@@ -124,15 +124,14 @@ module Karafka
|
|
|
124
124
|
|
|
125
125
|
# Builds and saves given consumer group
|
|
126
126
|
# @param group_id [String, Symbol] name for consumer group
|
|
127
|
-
|
|
128
|
-
def consumer_group(group_id, &block)
|
|
127
|
+
def consumer_group(group_id, &)
|
|
129
128
|
consumer_group = find { |cg| cg.name == group_id.to_s }
|
|
130
129
|
|
|
131
130
|
if consumer_group
|
|
132
|
-
Proxy.new(consumer_group, &
|
|
131
|
+
Proxy.new(consumer_group, &).target
|
|
133
132
|
else
|
|
134
133
|
consumer_group = ConsumerGroup.new(group_id.to_s)
|
|
135
|
-
self << Proxy.new(consumer_group, &
|
|
134
|
+
self << Proxy.new(consumer_group, &).target
|
|
136
135
|
end
|
|
137
136
|
end
|
|
138
137
|
|
|
@@ -140,8 +139,9 @@ module Karafka
|
|
|
140
139
|
# subscription group customization
|
|
141
140
|
# @param subscription_group_name [String, Symbol] subscription group id. When not provided,
|
|
142
141
|
# a random uuid will be used
|
|
143
|
-
# @param args [
|
|
142
|
+
# @param args [Hash] any extra arguments accepted by the subscription group builder
|
|
144
143
|
# @param block [Proc] further topics definitions
|
|
144
|
+
# @option args [String] :kafka optional kafka scope settings
|
|
145
145
|
def subscription_group(
|
|
146
146
|
subscription_group_name = SubscriptionGroup.id,
|
|
147
147
|
**args,
|
|
@@ -40,14 +40,13 @@ module Karafka
|
|
|
40
40
|
|
|
41
41
|
# Builds a topic representation inside of a current consumer group route
|
|
42
42
|
# @param name [String, Symbol] name of topic to which we want to subscribe
|
|
43
|
-
# @param block [Proc] block that we want to evaluate in the topic context
|
|
44
43
|
# @return [Karafka::Routing::Topic] newly built topic instance
|
|
45
|
-
def topic=(name, &
|
|
44
|
+
def topic=(name, &)
|
|
46
45
|
topic = Topic.new(name, self)
|
|
47
46
|
@topics << Proxy.new(
|
|
48
47
|
topic,
|
|
49
48
|
builder.defaults,
|
|
50
|
-
&
|
|
49
|
+
&
|
|
51
50
|
).target
|
|
52
51
|
built_topic = @topics.last
|
|
53
52
|
# We overwrite it conditionally in case it was not set by the user inline in the topic
|
|
@@ -59,13 +58,12 @@ module Karafka
|
|
|
59
58
|
# Assigns the current subscription group id based on the defined one and allows for further
|
|
60
59
|
# topic definition
|
|
61
60
|
# @param name [String, Symbol] name of the current subscription group
|
|
62
|
-
|
|
63
|
-
def subscription_group=(name = SubscriptionGroup.id, &block)
|
|
61
|
+
def subscription_group=(name = SubscriptionGroup.id, &)
|
|
64
62
|
# We cast it here, so the routing supports symbol based but that's anyhow later on
|
|
65
63
|
# validated as a string
|
|
66
64
|
@current_subscription_group_details = { name: name.to_s }
|
|
67
65
|
|
|
68
|
-
Proxy.new(self, &
|
|
66
|
+
Proxy.new(self, &)
|
|
69
67
|
|
|
70
68
|
# We need to reset the current subscription group after it is used, so it won't leak
|
|
71
69
|
# outside to other topics that would be defined without a defined subscription group
|
|
@@ -7,10 +7,8 @@ module Karafka
|
|
|
7
7
|
# Contract for single full route (consumer group + topics) validation.
|
|
8
8
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
9
9
|
configure do |config|
|
|
10
|
-
config.error_messages = YAML.
|
|
11
|
-
File.
|
|
12
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
13
|
-
)
|
|
10
|
+
config.error_messages = YAML.safe_load_file(
|
|
11
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
14
12
|
).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
|
|
15
13
|
end
|
|
16
14
|
|
|
@@ -39,21 +37,22 @@ module Karafka
|
|
|
39
37
|
topics_consumers[topic[:name]] << topic[:consumer]
|
|
40
38
|
end
|
|
41
39
|
|
|
42
|
-
next if topics_consumers.values.map(&:size).all?
|
|
40
|
+
next if topics_consumers.values.map(&:size).all?(1)
|
|
43
41
|
|
|
44
42
|
[[%i[topics], :many_consumers_same_topic]]
|
|
45
43
|
end
|
|
46
44
|
|
|
47
45
|
virtual do |data, errors|
|
|
48
46
|
next unless errors.empty?
|
|
49
|
-
next unless
|
|
47
|
+
next unless Karafka::App.config.strict_topics_namespacing
|
|
50
48
|
|
|
51
49
|
names = data.fetch(:topics).map { |topic| topic[:name] }
|
|
52
50
|
names_hash = names.each_with_object({}) { |n, h| h[n] = true }
|
|
53
51
|
error_occured = false
|
|
52
|
+
namespace_chars = ['.', '_'].freeze
|
|
54
53
|
names.each do |n|
|
|
55
54
|
# Skip topic names that are not namespaced
|
|
56
|
-
next unless n.chars.find { |c|
|
|
55
|
+
next unless n.chars.find { |c| namespace_chars.include?(c) }
|
|
57
56
|
|
|
58
57
|
if n.chars.include?('.')
|
|
59
58
|
# Check underscore styled topic
|
|
@@ -6,10 +6,8 @@ module Karafka
|
|
|
6
6
|
# Ensures that routing wide rules are obeyed
|
|
7
7
|
class Routing < Karafka::Contracts::Base
|
|
8
8
|
configure do |config|
|
|
9
|
-
config.error_messages = YAML.
|
|
10
|
-
File.
|
|
11
|
-
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
12
|
-
)
|
|
9
|
+
config.error_messages = YAML.safe_load_file(
|
|
10
|
+
File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
|
|
13
11
|
).fetch('en').fetch('validations').fetch('routing')
|
|
14
12
|
end
|
|
15
13
|
|