karafka 2.6.0.beta1 → 2.6.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +64 -27
- data/certs/expired.txt +2 -0
- data/config/locales/errors.yml +5 -0
- data/karafka.gemspec +1 -1
- data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes.rb +4 -2
- data/lib/karafka/active_job/dispatcher.rb +1 -2
- data/lib/karafka/admin/consumer_groups.rb +8 -2
- data/lib/karafka/admin/replication.rb +6 -3
- data/lib/karafka/admin/topics.rb +32 -12
- data/lib/karafka/admin.rb +155 -59
- data/lib/karafka/app.rb +1 -1
- data/lib/karafka/base_consumer.rb +33 -5
- data/lib/karafka/cli/console.rb +0 -1
- data/lib/karafka/cli/help.rb +0 -1
- data/lib/karafka/cli/info.rb +0 -1
- data/lib/karafka/cli/install.rb +0 -1
- data/lib/karafka/cli/server.rb +0 -1
- data/lib/karafka/cli/swarm.rb +0 -1
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +44 -7
- data/lib/karafka/connection/listener.rb +3 -3
- data/lib/karafka/connection/manager.rb +0 -1
- data/lib/karafka/connection/pauses_manager.rb +17 -0
- data/lib/karafka/connection/proxy.rb +45 -0
- data/lib/karafka/connection/status.rb +0 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +34 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb +19 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +0 -1
- data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +3 -2
- data/lib/karafka/instrumentation/critical_errors_listener.rb +85 -0
- data/lib/karafka/instrumentation/logger_listener.rb +2 -0
- data/lib/karafka/instrumentation/monitor.rb +2 -3
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +0 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +0 -1
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +4 -3
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +0 -2
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +181 -18
- data/lib/karafka/pro/admin/recovery/errors.rb +0 -1
- data/lib/karafka/pro/admin/recovery.rb +6 -4
- data/lib/karafka/pro/base_consumer.rb +5 -5
- data/lib/karafka/pro/cleaner/messages/metadata.rb +0 -1
- data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
- data/lib/karafka/pro/connection/manager.rb +7 -4
- data/lib/karafka/pro/encryption/messages/middleware.rb +5 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +4 -0
- data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb +63 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb +120 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb +88 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb +185 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/registry.rb +102 -0
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +40 -6
- data/lib/karafka/pro/iterator/tpl_builder.rb +27 -7
- data/lib/karafka/pro/iterator.rb +22 -7
- data/lib/karafka/pro/loader.rb +12 -0
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +2 -4
- data/lib/karafka/pro/processing/consumer_groups/collapser.rb +6 -9
- data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +21 -10
- data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +4 -4
- data/lib/karafka/pro/processing/consumer_groups/expansions_selector.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/filters/actions.rb +84 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +5 -6
- data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/consume_non_blocking.rb +2 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/eofed_non_blocking.rb +0 -2
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +2 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic_non_blocking.rb +0 -2
- data/lib/karafka/pro/processing/consumer_groups/jobs/revoked_non_blocking.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +5 -4
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +8 -10
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +3 -5
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +10 -9
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +11 -10
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom.rb +8 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +9 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +6 -9
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +4 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/base.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +56 -33
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +27 -14
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +65 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +0 -4
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +4 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +11 -6
- data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +2 -4
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb +0 -3
- data/lib/karafka/pro/processing/jobs_queue.rb +4 -1
- data/lib/karafka/pro/processing/schedulers/base.rb +4 -2
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +9 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +0 -1
- data/lib/karafka/pro/recurring_tasks/executor.rb +25 -13
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb +2 -2
- data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +0 -6
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +5 -4
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb +0 -6
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb +0 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +8 -8
- data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +5 -5
- data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +33 -13
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +52 -28
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +6 -5
- data/lib/karafka/pro/scheduled_messages/consumer.rb +7 -1
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +6 -1
- data/lib/karafka/pro/swarm/liveness_listener.rb +118 -13
- data/lib/karafka/pro.rb +0 -6
- data/lib/karafka/processing/consumer_groups/coordinator.rb +4 -5
- data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +6 -2
- data/lib/karafka/processing/consumer_groups/executor.rb +4 -4
- data/lib/karafka/processing/consumer_groups/executors_buffer.rb +2 -4
- data/lib/karafka/processing/consumer_groups/expansions_selector.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +14 -16
- data/lib/karafka/processing/consumer_groups/jobs/consume.rb +2 -5
- data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +2 -3
- data/lib/karafka/processing/consumer_groups/jobs/idle.rb +2 -2
- data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +2 -4
- data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +2 -2
- data/lib/karafka/processing/consumer_groups/partitioner.rb +3 -6
- data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +2 -4
- data/lib/karafka/processing/consumer_groups/strategies/base.rb +3 -5
- data/lib/karafka/processing/consumer_groups/strategies/default.rb +43 -13
- data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/mom.rb +2 -3
- data/lib/karafka/processing/consumer_groups/strategy_selector.rb +0 -2
- data/lib/karafka/processing/coordinator.rb +0 -2
- data/lib/karafka/processing/coordinators_buffer.rb +0 -2
- data/lib/karafka/processing/partitioner.rb +0 -2
- data/lib/karafka/processing/strategy_selector.rb +0 -2
- data/lib/karafka/processing/worker.rb +14 -6
- data/lib/karafka/processing/workers_pool.rb +21 -4
- data/lib/karafka/railtie.rb +0 -1
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +6 -5
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/topic.rb +0 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/topic.rb +4 -4
- data/lib/karafka/routing/topics.rb +21 -1
- data/lib/karafka/server.rb +9 -0
- data/lib/karafka/setup/config.rb +54 -10
- data/lib/karafka/setup/config_proxy.rb +2 -1
- data/lib/karafka/setup/contracts/config.rb +14 -0
- data/lib/karafka/swarm/node.rb +2 -2
- data/lib/karafka/version.rb +1 -2
- data/lib/karafka.rb +2 -2
- metadata +12 -4
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Lrj
|
|
@@ -31,14 +31,11 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Lrj
|
|
40
|
-
# Filtering enabled
|
|
41
|
-
# Long-Running Job enabled
|
|
37
|
+
# - Filtering enabled
|
|
38
|
+
# - Long-Running Job enabled
|
|
42
39
|
#
|
|
43
40
|
# In general aside from throttling this one will behave the same way as the Lrj
|
|
44
41
|
module Ftr
|
|
@@ -69,7 +66,7 @@ module Karafka
|
|
|
69
66
|
|
|
70
67
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
71
68
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
72
|
-
return unless coordinator.filter.
|
|
69
|
+
return unless coordinator.filter.skip?
|
|
73
70
|
elsif !revoked? && !coordinator.manual_seek?
|
|
74
71
|
# If not revoked and not throttled, we move to where we were suppose to and
|
|
75
72
|
# resume
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Lrj
|
|
40
|
-
# Long-Running Job enabled
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# Manual offset management enabled
|
|
37
|
+
# - Long-Running Job enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Manual offset management enabled
|
|
43
40
|
#
|
|
44
41
|
# It is really similar to the Lrj::Ftr but we do not mark anything as consumed
|
|
45
42
|
module FtrMom
|
|
@@ -67,7 +64,7 @@ module Karafka
|
|
|
67
64
|
|
|
68
65
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
69
66
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
70
|
-
return unless coordinator.filter.
|
|
67
|
+
return unless coordinator.filter.skip?
|
|
71
68
|
elsif !revoked? && !coordinator.manual_seek?
|
|
72
69
|
# If not revoked and not throttled, we move to where we were suppose to and
|
|
73
70
|
# resume
|
|
@@ -31,17 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
|
-
# Filtering related init strategies
|
|
40
36
|
module Lrj
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# LRJ enabled
|
|
43
|
-
# MoM enabled
|
|
44
|
-
# VPs enabled
|
|
37
|
+
# - Filtering enabled
|
|
38
|
+
# - LRJ enabled
|
|
39
|
+
# - MoM enabled
|
|
40
|
+
# - VPs enabled
|
|
45
41
|
module FtrMomVp
|
|
46
42
|
# Filtering + LRJ + Mom + VPs
|
|
47
43
|
FEATURES = %i[
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Lrj
|
|
40
|
-
# Long-Running Job enabled
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - Long-Running Job enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Virtual Partitions enabled
|
|
43
40
|
#
|
|
44
41
|
# Behaves same as non-VP because of the aggregated flow in the coordinator.
|
|
45
42
|
module FtrVp
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
# Namespace for all the LRJ starting strategies
|
|
40
37
|
module Lrj
|
|
41
|
-
# Long-Running Job enabled
|
|
42
|
-
# Manual offset management enabled
|
|
38
|
+
# - Long-Running Job enabled
|
|
39
|
+
# - Manual offset management enabled
|
|
43
40
|
module Mom
|
|
44
41
|
include Strategies::Default
|
|
45
42
|
|
|
@@ -31,16 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
|
-
# Namespace for all the LRJ starting strategies
|
|
40
36
|
module Lrj
|
|
41
|
-
# Long-Running Job enabled
|
|
42
|
-
# Manual offset management enabled
|
|
43
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - Long-Running Job enabled
|
|
38
|
+
# - Manual offset management enabled
|
|
39
|
+
# - Virtual Partitions enabled
|
|
44
40
|
module MomVp
|
|
45
41
|
include Strategies::Mom::Vp
|
|
46
42
|
include Strategies::Lrj::Mom
|
|
@@ -31,14 +31,11 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Lrj
|
|
40
|
-
# Long-Running Job enabled
|
|
41
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - Long-Running Job enabled
|
|
38
|
+
# - Virtual Partitions enabled
|
|
42
39
|
module Vp
|
|
43
40
|
# Same flow as the standard Lrj
|
|
44
41
|
include Strategies::Vp::Default
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
# Namespace for Mom starting strategies
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Mom
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Mom
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Mom
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
# VP starting strategies
|
|
@@ -71,9 +68,13 @@ module Karafka
|
|
|
71
68
|
# want to mark all but until the expected location
|
|
72
69
|
manager.mark_until(message, offset_metadata) if coordinator.finished?
|
|
73
70
|
|
|
74
|
-
|
|
71
|
+
# When there is no continuous stream of virtually marked offsets from the
|
|
72
|
+
# lowest registered one, there is no real offset to commit yet. This is not
|
|
73
|
+
# a failure, hence we need to report ownership truthfully: true as long as
|
|
74
|
+
# the assignment is still owned
|
|
75
|
+
return !revoked? unless manager.markable?
|
|
75
76
|
|
|
76
|
-
|
|
77
|
+
super(*manager.markable)
|
|
77
78
|
end
|
|
78
79
|
end
|
|
79
80
|
ensure
|
|
@@ -93,7 +94,11 @@ module Karafka
|
|
|
93
94
|
coordinator.synchronize do
|
|
94
95
|
manager.mark(message, offset_metadata)
|
|
95
96
|
manager.mark_until(message, offset_metadata) if coordinator.finished?
|
|
96
|
-
|
|
97
|
+
|
|
98
|
+
# No real offset to commit yet is not a failure, see `#mark_as_consumed`
|
|
99
|
+
return !revoked? unless manager.markable?
|
|
100
|
+
|
|
101
|
+
super(*manager.markable)
|
|
97
102
|
end
|
|
98
103
|
end
|
|
99
104
|
ensure
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Selector of appropriate processing strategy matching topic combinations
|
|
39
36
|
# When using Karafka Pro, there is a different set of strategies than for regular, as
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Uses the jobs queue API to lock (pause) and unlock (resume) operations of a given
|
|
39
36
|
# subscription group. It is abstracted away from jobs queue on this layer because we do
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module VirtualPartitions
|
|
39
36
|
module Distributors
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Processing components for virtual partitions
|
|
39
36
|
module VirtualPartitions
|
|
@@ -41,7 +38,8 @@ module Karafka
|
|
|
41
38
|
module Distributors
|
|
42
39
|
# Base class for all virtual partition distributors
|
|
43
40
|
class Base
|
|
44
|
-
# @param config
|
|
41
|
+
# @param config
|
|
42
|
+
# [Karafka::Pro::Routing::Features::ConsumerGroups::VirtualPartitions::Config]
|
|
45
43
|
def initialize(config)
|
|
46
44
|
@config = config
|
|
47
45
|
end
|
data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb
CHANGED
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module VirtualPartitions
|
|
39
36
|
module Distributors
|
|
@@ -151,7 +151,10 @@ module Karafka
|
|
|
151
151
|
@statistics[:waiting] -= @in_waiting[group_id].size
|
|
152
152
|
@in_waiting[group_id].clear
|
|
153
153
|
@locks[group_id].clear
|
|
154
|
-
|
|
154
|
+
# Recompute the async-locking fast-path flag from the remaining groups rather than
|
|
155
|
+
# forcing it off: this clear is per subscription group (run on that group's recovery)
|
|
156
|
+
# and must not disable async locks still held by other groups
|
|
157
|
+
@async_locking = @locks.any? { |_group_id, locks| locks.any? }
|
|
155
158
|
|
|
156
159
|
# We unlock it just in case it was blocked when clearing started
|
|
157
160
|
tick(group_id)
|
|
@@ -92,7 +92,8 @@ module Karafka
|
|
|
92
92
|
|
|
93
93
|
# Runs the shutdown jobs scheduling flow under a mutex
|
|
94
94
|
#
|
|
95
|
-
# @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Shutdown>] jobs for
|
|
95
|
+
# @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Shutdown>] jobs for
|
|
96
|
+
# scheduling
|
|
96
97
|
def on_schedule_shutdown(jobs_array)
|
|
97
98
|
@mutex.synchronize do
|
|
98
99
|
schedule_shutdown(jobs_array)
|
|
@@ -101,7 +102,8 @@ module Karafka
|
|
|
101
102
|
|
|
102
103
|
# Runs the idle jobs scheduling flow under a mutex
|
|
103
104
|
#
|
|
104
|
-
# @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Idle>] jobs for
|
|
105
|
+
# @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Idle>] jobs for
|
|
106
|
+
# scheduling
|
|
105
107
|
def on_schedule_idle(jobs_array)
|
|
106
108
|
@mutex.synchronize do
|
|
107
109
|
schedule_idle(jobs_array)
|
|
@@ -31,7 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Namespace for Pro schedulers
|
|
35
34
|
module Schedulers
|
|
36
35
|
# Optimizes scheduler that takes into consideration of execution time needed to process
|
|
37
36
|
# messages from given topics partitions. It uses the non-preemptive LJF algorithm
|
|
@@ -109,7 +108,8 @@ module Karafka
|
|
|
109
108
|
messages = job.messages
|
|
110
109
|
message = messages.first
|
|
111
110
|
|
|
112
|
-
perf_tracker.processing_time_p95(message.topic, message.partition) *
|
|
111
|
+
perf_tracker.processing_time_p95(job.group_id, message.topic, message.partition) *
|
|
112
|
+
messages.size
|
|
113
113
|
else
|
|
114
114
|
# LJF will set first the most expensive, but we want to run the zero cost jobs
|
|
115
115
|
# related to the lifecycle always first. That is why we "emulate" that they
|
|
@@ -71,9 +71,16 @@ module Karafka
|
|
|
71
71
|
message.offset - 1
|
|
72
72
|
)
|
|
73
73
|
when "command"
|
|
74
|
-
|
|
74
|
+
# During replaying commands are only buffered. Applying them immediately would be
|
|
75
|
+
# pointless as the end-of-replay synchronization with the last stored schedule
|
|
76
|
+
# state would overwrite their effects. They are applied after the synchronization
|
|
77
|
+
if @executor.replaying?
|
|
78
|
+
@executor.replay_command(payload)
|
|
79
|
+
|
|
80
|
+
next
|
|
81
|
+
end
|
|
75
82
|
|
|
76
|
-
|
|
83
|
+
@executor.apply_command(payload)
|
|
77
84
|
|
|
78
85
|
# Execute on each incoming command to have nice latency but only after replaying
|
|
79
86
|
# During replaying we should not execute because there may be more state changes
|
|
@@ -76,6 +76,14 @@ module Karafka
|
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
+
# Buffers a command received during replay, so it can be applied once the in-memory
|
|
80
|
+
# state is synchronized with the last stored schedule state. Commands applied directly
|
|
81
|
+
# during replay would have their effects overwritten by this synchronization
|
|
82
|
+
# @param command_hash [Hash] deserialized command data
|
|
83
|
+
def replay_command(command_hash)
|
|
84
|
+
@catchup_commands << command_hash
|
|
85
|
+
end
|
|
86
|
+
|
|
79
87
|
# Updates the catchup state
|
|
80
88
|
# @param schedule_hash [Hash] deserialized schedule hash hash
|
|
81
89
|
def update_state(schedule_hash)
|
|
@@ -100,24 +108,28 @@ module Karafka
|
|
|
100
108
|
return
|
|
101
109
|
end
|
|
102
110
|
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
if @catchup_schedule
|
|
106
|
-
|
|
111
|
+
# Commands may have been received without any schedule state stored in Kafka. In such
|
|
112
|
+
# cases there is nothing to synchronize and only the commands are applied
|
|
113
|
+
if @catchup_schedule
|
|
114
|
+
# If the schedule version we have in Kafka is higher than ours, we cannot proceed
|
|
115
|
+
# This prevents us from applying older changes to a new schedule
|
|
116
|
+
if Gem::Version.new(@catchup_schedule[:schedule_version]) > Gem::Version.new(schedule.version)
|
|
117
|
+
@incompatible = true
|
|
107
118
|
|
|
108
|
-
|
|
109
|
-
|
|
119
|
+
return
|
|
120
|
+
end
|
|
110
121
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
122
|
+
# Now we can synchronize the in-memory state based on the last state stored in Kafka
|
|
123
|
+
schedule.each do |task|
|
|
124
|
+
stored_task = @catchup_schedule[:tasks][task.id.to_sym]
|
|
114
125
|
|
|
115
|
-
|
|
126
|
+
next unless stored_task
|
|
116
127
|
|
|
117
|
-
|
|
118
|
-
|
|
128
|
+
stored_previous_time = stored_task[:previous_time]
|
|
129
|
+
task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
|
|
119
130
|
|
|
120
|
-
|
|
131
|
+
stored_task[:enabled] ? task.enable : task.disable
|
|
132
|
+
end
|
|
121
133
|
end
|
|
122
134
|
|
|
123
135
|
@catchup_commands.each do |cmd|
|
data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb
CHANGED
|
@@ -33,8 +33,6 @@ module Karafka
|
|
|
33
33
|
module Routing
|
|
34
34
|
module Features
|
|
35
35
|
module ConsumerGroups
|
|
36
|
-
# Alterations to the direct assignments that allow us to do stable direct assignments
|
|
37
|
-
# without working with consumer groups dynamic assignments
|
|
38
36
|
class DirectAssignments < Base
|
|
39
37
|
# Extension allowing us to select correct subscriptions and assignments based on the
|
|
40
38
|
# expanded routing setup
|
|
@@ -36,8 +36,8 @@ module Karafka
|
|
|
36
36
|
class Multiplexing < Base
|
|
37
37
|
# Allows for multiplexing setup inside a consumer group definition
|
|
38
38
|
module Proxy
|
|
39
|
-
# @param min [Integer, nil] min multiplexing count or nil to set it to max,
|
|
40
|
-
# disabling dynamic multiplexing
|
|
39
|
+
# @param min [Integer, nil] min multiplexing count or nil to set it to max,
|
|
40
|
+
# effectively disabling dynamic multiplexing
|
|
41
41
|
# @param max [Integer] max multiplexing count
|
|
42
42
|
# @param boot [Integer] how many listeners should we start during boot by default
|
|
43
43
|
# @param scale_delay [Integer] number of ms of delay before applying any scale
|
|
@@ -33,12 +33,6 @@ module Karafka
|
|
|
33
33
|
module Routing
|
|
34
34
|
module Features
|
|
35
35
|
module ConsumerGroups
|
|
36
|
-
# This feature allows for saving and retrieving offset metadata with custom deserialization
|
|
37
|
-
# support. It allows for storing extra data during commits that can be then used to alter
|
|
38
|
-
# the processing flow after a rebalance.
|
|
39
|
-
#
|
|
40
|
-
# @note Because this feature has zero performance impact and makes no queries to Kafka
|
|
41
|
-
# unless requested, it is always enabled.
|
|
42
36
|
class OffsetMetadata < Base
|
|
43
37
|
# Empty string not to create it on each deserialization
|
|
44
38
|
EMPTY_STRING = ""
|
|
@@ -50,12 +50,13 @@ module Karafka
|
|
|
50
50
|
# groups that will be created)
|
|
51
51
|
# @param partitioner [nil, #call] nil or callable partitioner
|
|
52
52
|
# @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
|
|
53
|
-
# reducer to achieve enhanced parallelization when the default reducer is not
|
|
53
|
+
# reducer to achieve enhanced parallelization when the default reducer is not
|
|
54
|
+
# enough.
|
|
54
55
|
# @param merge_key [String] key used to build the parallel segment consumer groups
|
|
55
56
|
#
|
|
56
|
-
# @note This method is an assignor but the API is actually via the
|
|
57
|
-
# method. Our `Routing::Proxy` normalizes that the way we want
|
|
58
|
-
# for the end users.
|
|
57
|
+
# @note This method is an assignor but the API is actually via the
|
|
58
|
+
# `#parallel_segments` method. Our `Routing::Proxy` normalizes that the way we want
|
|
59
|
+
# to have it exposed for the end users.
|
|
59
60
|
def parallel_segments=(
|
|
60
61
|
count: 1,
|
|
61
62
|
partitioner: nil,
|
|
@@ -33,12 +33,6 @@ module Karafka
|
|
|
33
33
|
module Routing
|
|
34
34
|
module Features
|
|
35
35
|
module ConsumerGroups
|
|
36
|
-
# Holds pattern info reference
|
|
37
|
-
# Type is set to:
|
|
38
|
-
# `:regular` - in case patterns are not used and topic is just a regular existing topic
|
|
39
|
-
# matched directly based on the name
|
|
40
|
-
# `:discovered` - in case it is a real topic on which we started to listed
|
|
41
|
-
# `:matcher` - represents a regular expression used by librdkafka
|
|
42
36
|
class Patterns < Base
|
|
43
37
|
# Config for pattern based topic Only pattern related topics are active in this context
|
|
44
38
|
Config = Struct.new(
|
|
@@ -42,7 +42,8 @@ module Karafka
|
|
|
42
42
|
@patterns = Patterns.new([])
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
# @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns]
|
|
45
|
+
# @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns]
|
|
46
|
+
# created patterns
|
|
46
47
|
def patterns
|
|
47
48
|
@patterns
|
|
48
49
|
end
|
|
@@ -71,7 +71,8 @@ module Karafka
|
|
|
71
71
|
|
|
72
72
|
# Adds the discovered topic into the routing
|
|
73
73
|
#
|
|
74
|
-
# @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern]
|
|
74
|
+
# @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern]
|
|
75
|
+
# matched pattern
|
|
75
76
|
# @param discovered_topic [String] topic that we discovered that should be part of the
|
|
76
77
|
# routing from now on.
|
|
77
78
|
# @param sg_topics [Array<Karafka::Routing::Topic>]
|
|
@@ -100,7 +100,8 @@ module Karafka
|
|
|
100
100
|
# prevent it from running but internally in this component we need to ensure, that
|
|
101
101
|
# prior to the validations we operate on a regexp
|
|
102
102
|
#
|
|
103
|
-
# @return [Regexp] returns a regexp always even if what we've received was not a
|
|
103
|
+
# @return [Regexp] returns a regexp always even if what we've received was not a
|
|
104
|
+
# regexp
|
|
104
105
|
def safe_regexp
|
|
105
106
|
# This regexp will never match anything
|
|
106
107
|
regexp.is_a?(Regexp) ? regexp : /$a/
|
|
@@ -47,15 +47,15 @@ module Karafka
|
|
|
47
47
|
#
|
|
48
48
|
# @param topic_name [String] topic name
|
|
49
49
|
# @return [Karafka::Routing::Topic]
|
|
50
|
-
# @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see
|
|
51
|
-
# please create an issue.
|
|
50
|
+
# @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see
|
|
51
|
+
# it, please create an issue.
|
|
52
52
|
#
|
|
53
|
-
# @note This method should not be used in context of finding multiple missing topics
|
|
54
|
-
# loops because it catches exceptions and attempts to expand routes. If this is
|
|
55
|
-
# in a loop for lookups on thousands of topics with detector expansion, this
|
|
56
|
-
# be slow. It should be used in the context where newly discovered topics are
|
|
57
|
-
# and should by design match a pattern. For quick lookups on batches of
|
|
58
|
-
# is recommended to use a custom built lookup with conditional expander.
|
|
53
|
+
# @note This method should not be used in context of finding multiple missing topics
|
|
54
|
+
# in loops because it catches exceptions and attempts to expand routes. If this is
|
|
55
|
+
# used in a loop for lookups on thousands of topics with detector expansion, this
|
|
56
|
+
# may be slow. It should be used in the context where newly discovered topics are
|
|
57
|
+
# found and should by design match a pattern. For quick lookups on batches of
|
|
58
|
+
# topics, it is recommended to use a custom built lookup with conditional expander.
|
|
59
59
|
def find(topic_name)
|
|
60
60
|
super
|
|
61
61
|
rescue Karafka::Errors::TopicNotFoundError
|
|
@@ -49,12 +49,12 @@ module Karafka
|
|
|
49
49
|
# poll where messages were not received.
|
|
50
50
|
# @param active [Boolean] should ticking happen for this topic assignments.
|
|
51
51
|
# @param interval [Integer] minimum interval to run periodic jobs on given topic.
|
|
52
|
-
# @param during_pause [Boolean, nil] Should periodic jobs run when partition is
|
|
53
|
-
# It is set to `nil` by default allowing for detection when this value is
|
|
54
|
-
# configured but should be built dynamically based on LRJ status.
|
|
52
|
+
# @param during_pause [Boolean, nil] Should periodic jobs run when partition is
|
|
53
|
+
# paused. It is set to `nil` by default allowing for detection when this value is
|
|
54
|
+
# not configured but should be built dynamically based on LRJ status.
|
|
55
55
|
# @param during_retry [Boolean, nil] Should we run when there was an error and we are
|
|
56
|
-
# in a retry flow. Please note that for this to work, `during_pause` also needs to
|
|
57
|
-
# set to true as errors retry happens after pause.
|
|
56
|
+
# in a retry flow. Please note that for this to work, `during_pause` also needs to
|
|
57
|
+
# be set to true as errors retry happens after pause.
|
|
58
58
|
def periodic_job(
|
|
59
59
|
active = false,
|
|
60
60
|
interval: nil,
|