karafka 2.5.5 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/LICENSE-COMM +4 -0
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +3 -3
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +1 -1
- data/lib/karafka/admin/consumer_groups.rb +8 -8
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +21 -21
- data/lib/karafka/admin/topics.rb +6 -6
- data/lib/karafka/admin.rb +4 -5
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +6 -1
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/interval_runner.rb +4 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +76 -63
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +33 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/builders/batch_metadata.rb +4 -2
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +10 -0
- data/lib/karafka/pro/cli/contracts/server.rb +12 -12
- data/lib/karafka/pro/cli/parallel_segments/base.rb +4 -4
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +3 -3
- data/lib/karafka/pro/cli/parallel_segments.rb +7 -7
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +14 -14
- data/lib/karafka/pro/encryption/contracts/config.rb +2 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +2 -2
- data/lib/karafka/pro/encryption/messages/parser.rb +2 -2
- data/lib/karafka/pro/encryption/setup/config.rb +2 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -2
- data/lib/karafka/pro/iterator.rb +1 -1
- data/lib/karafka/pro/loader.rb +2 -1
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +1 -1
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +24 -14
- data/lib/karafka/pro/processing/filters/base.rb +1 -1
- data/lib/karafka/pro/processing/filters/delayer.rb +2 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +1 -1
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +1 -1
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +6 -6
- data/lib/karafka/pro/processing/partitioner.rb +3 -3
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +6 -5
- data/lib/karafka/pro/processing/piping/consumer.rb +7 -7
- data/lib/karafka/pro/processing/schedulers/base.rb +5 -5
- data/lib/karafka/pro/processing/schedulers/default.rb +5 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +6 -3
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +2 -2
- data/lib/karafka/pro/processing/strategies/default.rb +22 -22
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +7 -7
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +6 -3
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +2 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +6 -3
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +1 -1
- data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +5 -5
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +3 -3
- data/lib/karafka/pro/recurring_tasks/task.rb +4 -4
- data/lib/karafka/pro/recurring_tasks.rb +4 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +5 -5
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/offset_metadata.rb +1 -1
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +5 -5
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +3 -3
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +1 -1
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +7 -7
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +13 -13
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/swarm.rb +1 -1
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +7 -7
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +10 -10
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +4 -4
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +5 -5
- data/lib/karafka/pro/scheduled_messages/proxy.rb +8 -8
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -2
- data/lib/karafka/pro/scheduled_messages/state.rb +1 -1
- data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -2
- data/lib/karafka/pro/scheduled_messages.rb +2 -2
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -2
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/jobs_queue.rb +10 -0
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +14 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +8 -37
- data/lib/karafka/swarm/producer_replacer.rb +110 -0
- data/lib/karafka/swarm/supervisor.rb +9 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -3
|
@@ -31,7 +31,7 @@ module Karafka
|
|
|
31
31
|
class << self
|
|
32
32
|
# @return [Schedule, nil] current defined schedule or nil if not defined
|
|
33
33
|
def schedule
|
|
34
|
-
@schedule || define(
|
|
34
|
+
@schedule || define("0.0.0") { nil }
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
# Simplified API for schedules definitions and validates the tasks data
|
|
@@ -44,14 +44,14 @@ module Karafka
|
|
|
44
44
|
# MailingJob.perform_async
|
|
45
45
|
# end
|
|
46
46
|
# end
|
|
47
|
-
def define(version =
|
|
47
|
+
def define(version = "1.0.0", &)
|
|
48
48
|
@schedule = Schedule.new(version: version)
|
|
49
49
|
@schedule.instance_exec(&)
|
|
50
50
|
|
|
51
51
|
@schedule.each do |task|
|
|
52
52
|
Contracts::Task.new.validate!(
|
|
53
53
|
task.to_h,
|
|
54
|
-
scope: [
|
|
54
|
+
scope: ["recurring_tasks", task.id]
|
|
55
55
|
)
|
|
56
56
|
end
|
|
57
57
|
|
|
@@ -88,7 +88,7 @@ module Karafka
|
|
|
88
88
|
)
|
|
89
89
|
|
|
90
90
|
# Published after task is successfully executed
|
|
91
|
-
Karafka.monitor.notifications_bus.register_event(
|
|
91
|
+
Karafka.monitor.notifications_bus.register_event("recurring_tasks.task.executed")
|
|
92
92
|
|
|
93
93
|
# Initialize empty dummy schedule, so we always have one and so we do not have to
|
|
94
94
|
# deal with a case where there is no schedule
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:adaptive_iterator) do
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:dead_letter_queue) do
|
|
@@ -48,7 +48,7 @@ module Karafka
|
|
|
48
48
|
super(**args).tap do |config|
|
|
49
49
|
# If explicit strategy is not provided, use the default approach from OSS
|
|
50
50
|
config.strategy = strategy || lambda do |_errors_tracker, attempt|
|
|
51
|
-
attempt > config.max_retries ? :dispatch : :retry
|
|
51
|
+
(attempt > config.max_retries) ? :dispatch : :retry
|
|
52
52
|
end
|
|
53
53
|
end
|
|
54
54
|
end
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:delaying) do
|
|
@@ -30,8 +30,8 @@ module Karafka
|
|
|
30
30
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
31
31
|
configure do |config|
|
|
32
32
|
config.error_messages = YAML.safe_load_file(
|
|
33
|
-
File.join(Karafka.gem_root,
|
|
34
|
-
).fetch(
|
|
33
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
34
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
|
|
35
35
|
|
|
36
36
|
virtual do |data, errors|
|
|
37
37
|
next unless errors.empty?
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:direct_assignments) do
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:expiring) do
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:filtering) do
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested :inline_insights do
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:long_running_job) do
|
|
@@ -41,7 +41,7 @@ module Karafka
|
|
|
41
41
|
@long_running_job ||= Config.new(active: active)
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
alias_method :long_running, :long_running_job
|
|
45
45
|
|
|
46
46
|
# @return [Boolean] is a given job on a topic a long-running one
|
|
47
47
|
def long_running_job?
|
|
@@ -35,8 +35,8 @@ module Karafka
|
|
|
35
35
|
class Topic < Karafka::Contracts::Base
|
|
36
36
|
configure do |config|
|
|
37
37
|
config.error_messages = YAML.safe_load_file(
|
|
38
|
-
File.join(Karafka.gem_root,
|
|
39
|
-
).fetch(
|
|
38
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
39
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
nested(:subscription_group_details) do
|
|
@@ -43,15 +43,15 @@ module Karafka
|
|
|
43
43
|
#
|
|
44
44
|
# @param _config [Karafka::Core::Configurable::Node] app config
|
|
45
45
|
def post_setup(_config)
|
|
46
|
-
Karafka::App.monitor.subscribe(
|
|
46
|
+
Karafka::App.monitor.subscribe("app.running") do
|
|
47
47
|
# Do not install the manager and listener to control multiplexing unless there is
|
|
48
48
|
# multiplexing enabled and it is dynamic.
|
|
49
49
|
# We only need to control multiplexing when it is in a dynamic state
|
|
50
50
|
next unless Karafka::App
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
51
|
+
.subscription_groups
|
|
52
|
+
.values
|
|
53
|
+
.flat_map(&:itself)
|
|
54
|
+
.any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
|
|
55
55
|
|
|
56
56
|
# Subscribe for events and possibility to manage via the Pro connection manager
|
|
57
57
|
# that supports multiplexing
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:offset_metadata) do
|
|
@@ -32,7 +32,7 @@ module Karafka
|
|
|
32
32
|
# unless requested, it is always enabled.
|
|
33
33
|
class OffsetMetadata < Base
|
|
34
34
|
# Empty string not to create it on each deserialization
|
|
35
|
-
EMPTY_STRING =
|
|
35
|
+
EMPTY_STRING = ""
|
|
36
36
|
|
|
37
37
|
# Default deserializer just ensures we always get a string as without metadata by
|
|
38
38
|
# default it would be nil
|
|
@@ -31,7 +31,7 @@ module Karafka
|
|
|
31
31
|
#
|
|
32
32
|
# @param _config [Karafka::Core::Configurable::Node] app config
|
|
33
33
|
def post_setup(_config)
|
|
34
|
-
Karafka::App.monitor.subscribe(
|
|
34
|
+
Karafka::App.monitor.subscribe("app.running") do
|
|
35
35
|
# Initialize the tracker prior to becoming multi-threaded
|
|
36
36
|
Karafka::Processing::InlineInsights::Tracker.instance
|
|
37
37
|
|
|
@@ -51,7 +51,7 @@ module Karafka
|
|
|
51
51
|
count: 1,
|
|
52
52
|
partitioner: nil,
|
|
53
53
|
reducer: nil,
|
|
54
|
-
merge_key:
|
|
54
|
+
merge_key: "-parallel-"
|
|
55
55
|
)
|
|
56
56
|
@parallel_segments ||= Config.new(
|
|
57
57
|
active: count > 1,
|
|
@@ -73,10 +73,10 @@ module Karafka
|
|
|
73
73
|
return @segment_id if @segment_id
|
|
74
74
|
|
|
75
75
|
@segment_id = if parallel_segments?
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
name.split(parallel_segments.merge_key).last.to_i
|
|
77
|
+
else
|
|
78
|
+
-1
|
|
79
|
+
end
|
|
80
80
|
end
|
|
81
81
|
|
|
82
82
|
# @return [String] original segment consumer group name
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
|
|
36
36
|
|
|
37
37
|
nested(:parallel_segments) do
|
|
38
38
|
required(:active) { |val| [true, false].include?(val) }
|
|
@@ -30,8 +30,8 @@ module Karafka
|
|
|
30
30
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
31
31
|
configure do |config|
|
|
32
32
|
config.error_messages = YAML.safe_load_file(
|
|
33
|
-
File.join(Karafka.gem_root,
|
|
34
|
-
).fetch(
|
|
33
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
34
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
|
|
35
35
|
|
|
36
36
|
required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
|
|
37
37
|
|
|
@@ -31,13 +31,13 @@ module Karafka
|
|
|
31
31
|
class Pattern < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("pattern")
|
|
36
36
|
|
|
37
37
|
required(:regexp) { |val| val.is_a?(Regexp) }
|
|
38
38
|
|
|
39
39
|
required(:regexp_string) do |val|
|
|
40
|
-
val.is_a?(String) && val.start_with?(
|
|
40
|
+
val.is_a?(String) && val.start_with?("^") && val.size >= 2
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
required(:name) do |val|
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:patterns) do
|
|
@@ -39,7 +39,7 @@ module Karafka
|
|
|
39
39
|
# @return [String] subscription name or the regexp string representing matching of
|
|
40
40
|
# new topics that should be detected.
|
|
41
41
|
def subscription_name
|
|
42
|
-
patterns.active? && patterns.matcher? ? patterns.pattern.regexp_string : super
|
|
42
|
+
(patterns.active? && patterns.matcher?) ? patterns.pattern.regexp_string : super
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
# @param active [Boolean] is this topic active member of patterns
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
|
|
37
37
|
# Validate the nested pausing configuration
|
|
38
38
|
# Both old setters and new pause() method update the pausing config,
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:periodic_job) do
|
|
@@ -37,7 +37,7 @@ module Karafka
|
|
|
37
37
|
|
|
38
38
|
# We only require zlib when we decide to run recurring tasks because it is not needed
|
|
39
39
|
# otherwise.
|
|
40
|
-
require
|
|
40
|
+
require "zlib"
|
|
41
41
|
ensure_fugit_availability!
|
|
42
42
|
|
|
43
43
|
tasks_cfg = App.config.recurring_tasks
|
|
@@ -62,7 +62,7 @@ module Karafka
|
|
|
62
62
|
consumer_persistence(true)
|
|
63
63
|
|
|
64
64
|
# This needs to be enabled for the eof to work correctly
|
|
65
|
-
kafka(
|
|
65
|
+
kafka("enable.partition.eof": true, inherit: true)
|
|
66
66
|
eofed(true)
|
|
67
67
|
|
|
68
68
|
# Favour latency. This is a low traffic topic that only accepts user initiated
|
|
@@ -80,8 +80,8 @@ module Karafka
|
|
|
80
80
|
|
|
81
81
|
# Keep older data for a day and compact to the last state available
|
|
82
82
|
config(
|
|
83
|
-
|
|
84
|
-
|
|
83
|
+
"cleanup.policy": "compact,delete",
|
|
84
|
+
"retention.ms": 86_400_000
|
|
85
85
|
)
|
|
86
86
|
|
|
87
87
|
# This is the core of execution. Since we're producers of states, we need a way
|
|
@@ -108,8 +108,8 @@ module Karafka
|
|
|
108
108
|
# Keep cron logs of executions for a week and after that remove. Week should be
|
|
109
109
|
# enough and should not produce too much data.
|
|
110
110
|
config(
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
"cleanup.policy": "delete",
|
|
112
|
+
"retention.ms": 604_800_000
|
|
113
113
|
)
|
|
114
114
|
end
|
|
115
115
|
|
|
@@ -122,7 +122,7 @@ module Karafka
|
|
|
122
122
|
def ensure_fugit_availability!
|
|
123
123
|
return if Object.const_defined?(:Fugit)
|
|
124
124
|
|
|
125
|
-
require
|
|
125
|
+
require "fugit"
|
|
126
126
|
rescue LoadError
|
|
127
127
|
raise(
|
|
128
128
|
Karafka::Errors::DependencyConstraintsError,
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:recurring_tasks) do
|
|
@@ -39,7 +39,7 @@ module Karafka
|
|
|
39
39
|
return unless topic_name
|
|
40
40
|
|
|
41
41
|
# Load zlib only if user enables scheduled messages
|
|
42
|
-
require
|
|
42
|
+
require "zlib"
|
|
43
43
|
|
|
44
44
|
# We set it to 5 so we have enough space to handle more events. All related topics
|
|
45
45
|
# should have same partition count.
|
|
@@ -72,8 +72,8 @@ module Karafka
|
|
|
72
72
|
|
|
73
73
|
# This needs to be enabled for the eof to work correctly
|
|
74
74
|
kafka(
|
|
75
|
-
|
|
76
|
-
|
|
75
|
+
"enable.partition.eof": true,
|
|
76
|
+
"auto.offset.reset": "earliest",
|
|
77
77
|
inherit: true
|
|
78
78
|
)
|
|
79
79
|
eofed(true)
|
|
@@ -91,13 +91,13 @@ module Karafka
|
|
|
91
91
|
partitions: default_partitions,
|
|
92
92
|
# Will ensure, that after tombstone is present, given scheduled message, that
|
|
93
93
|
# has been dispatched is removed by Kafka
|
|
94
|
-
|
|
94
|
+
"cleanup.policy": "compact",
|
|
95
95
|
# When 10% or more dispatches are done, compact data
|
|
96
|
-
|
|
96
|
+
"min.cleanable.dirty.ratio": 0.1,
|
|
97
97
|
# Frequent segment rotation to support intense compaction
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
"segment.ms": 3_600_000,
|
|
99
|
+
"delete.retention.ms": 3_600_000,
|
|
100
|
+
"segment.bytes": 52_428_800
|
|
101
101
|
)
|
|
102
102
|
|
|
103
103
|
# This is the core of execution. Since we dispatch data in time intervals, we
|
|
@@ -122,11 +122,11 @@ module Karafka
|
|
|
122
122
|
target.scheduled_messages(true)
|
|
123
123
|
config(
|
|
124
124
|
partitions: default_partitions,
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
125
|
+
"cleanup.policy": "compact",
|
|
126
|
+
"min.cleanable.dirty.ratio": 0.1,
|
|
127
|
+
"segment.ms": 3_600_000,
|
|
128
|
+
"delete.retention.ms": 3_600_000,
|
|
129
|
+
"segment.bytes": 52_428_800
|
|
130
130
|
)
|
|
131
131
|
deserializers(
|
|
132
132
|
payload: msg_cfg.deserializers.payload
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:scheduled_messages) do
|
|
@@ -36,8 +36,8 @@ module Karafka
|
|
|
36
36
|
class Routing < Karafka::Contracts::Base
|
|
37
37
|
configure do |config|
|
|
38
38
|
config.error_messages = YAML.safe_load_file(
|
|
39
|
-
File.join(Karafka.gem_root,
|
|
40
|
-
).fetch(
|
|
39
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
40
|
+
).fetch("en").fetch("validations").fetch("routing")
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
# Validates that each node has at least one assignment.
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:swarm) do
|
|
@@ -33,7 +33,7 @@ module Karafka
|
|
|
33
33
|
#
|
|
34
34
|
# @param config [Karafka::Core::Configurable::Node]
|
|
35
35
|
def post_setup(config)
|
|
36
|
-
config.monitor.subscribe(
|
|
36
|
+
config.monitor.subscribe("app.before_warmup") do
|
|
37
37
|
Contracts::Routing.new.validate!(
|
|
38
38
|
config.internal.routing.builder,
|
|
39
39
|
scope: %w[swarm]
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:throttling) do
|
|
@@ -43,13 +43,13 @@ module Karafka
|
|
|
43
43
|
# @return [Object] distributor instance for the current distribution
|
|
44
44
|
def distributor
|
|
45
45
|
@distributor ||= case distribution
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
46
|
+
when :balanced
|
|
47
|
+
Processing::VirtualPartitions::Distributors::Balanced.new(self)
|
|
48
|
+
when :consistent
|
|
49
|
+
Processing::VirtualPartitions::Distributors::Consistent.new(self)
|
|
50
|
+
else
|
|
51
|
+
raise Karafka::Errors::UnsupportedCaseError, distribution
|
|
52
|
+
end
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
55
|
end
|
|
@@ -31,8 +31,8 @@ module Karafka
|
|
|
31
31
|
class Topic < Karafka::Contracts::Base
|
|
32
32
|
configure do |config|
|
|
33
33
|
config.error_messages = YAML.safe_load_file(
|
|
34
|
-
File.join(Karafka.gem_root,
|
|
35
|
-
).fetch(
|
|
34
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
35
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
nested(:virtual_partitions) do
|