karafka 2.6.0.beta1 → 2.6.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +64 -27
- data/certs/expired.txt +2 -0
- data/config/locales/errors.yml +5 -0
- data/karafka.gemspec +1 -1
- data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -1
- data/lib/karafka/active_job/current_attributes/loading.rb +1 -1
- data/lib/karafka/active_job/current_attributes.rb +4 -2
- data/lib/karafka/active_job/dispatcher.rb +1 -2
- data/lib/karafka/admin/consumer_groups.rb +8 -2
- data/lib/karafka/admin/replication.rb +6 -3
- data/lib/karafka/admin/topics.rb +32 -12
- data/lib/karafka/admin.rb +155 -59
- data/lib/karafka/app.rb +1 -1
- data/lib/karafka/base_consumer.rb +33 -5
- data/lib/karafka/cli/console.rb +0 -1
- data/lib/karafka/cli/help.rb +0 -1
- data/lib/karafka/cli/info.rb +0 -1
- data/lib/karafka/cli/install.rb +0 -1
- data/lib/karafka/cli/server.rb +0 -1
- data/lib/karafka/cli/swarm.rb +0 -1
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +44 -7
- data/lib/karafka/connection/listener.rb +3 -3
- data/lib/karafka/connection/manager.rb +0 -1
- data/lib/karafka/connection/pauses_manager.rb +17 -0
- data/lib/karafka/connection/proxy.rb +45 -0
- data/lib/karafka/connection/status.rb +0 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +34 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/decorator.rb +19 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +0 -1
- data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +3 -2
- data/lib/karafka/instrumentation/critical_errors_listener.rb +85 -0
- data/lib/karafka/instrumentation/logger_listener.rb +2 -0
- data/lib/karafka/instrumentation/monitor.rb +2 -3
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +0 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +0 -1
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +4 -3
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +0 -2
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +181 -18
- data/lib/karafka/pro/admin/recovery/errors.rb +0 -1
- data/lib/karafka/pro/admin/recovery.rb +6 -4
- data/lib/karafka/pro/base_consumer.rb +5 -5
- data/lib/karafka/pro/cleaner/messages/metadata.rb +0 -1
- data/lib/karafka/pro/cli/parallel_segments/base.rb +1 -2
- data/lib/karafka/pro/connection/manager.rb +7 -4
- data/lib/karafka/pro/encryption/messages/middleware.rb +5 -0
- data/lib/karafka/pro/encryption/messages/parser.rb +4 -0
- data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb +63 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb +120 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb +88 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb +185 -0
- data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/registry.rb +102 -0
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +40 -6
- data/lib/karafka/pro/iterator/tpl_builder.rb +27 -7
- data/lib/karafka/pro/iterator.rb +22 -7
- data/lib/karafka/pro/loader.rb +12 -0
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +2 -4
- data/lib/karafka/pro/processing/consumer_groups/collapser.rb +6 -9
- data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +21 -10
- data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +4 -4
- data/lib/karafka/pro/processing/consumer_groups/expansions_selector.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/filters/actions.rb +84 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +5 -6
- data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +1 -4
- data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/consume_non_blocking.rb +2 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/eofed_non_blocking.rb +0 -2
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +2 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic_non_blocking.rb +0 -2
- data/lib/karafka/pro/processing/consumer_groups/jobs/revoked_non_blocking.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +5 -4
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +8 -10
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +3 -5
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +10 -9
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +11 -10
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom.rb +8 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +9 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +6 -9
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +4 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/base.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +56 -33
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +27 -14
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb +5 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +65 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +0 -4
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +4 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +4 -8
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +3 -6
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +3 -7
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb +2 -5
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +11 -6
- data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +0 -3
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +2 -4
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb +0 -3
- data/lib/karafka/pro/processing/jobs_queue.rb +4 -1
- data/lib/karafka/pro/processing/schedulers/base.rb +4 -2
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +9 -2
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +0 -1
- data/lib/karafka/pro/recurring_tasks/executor.rb +25 -13
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb +2 -2
- data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb +0 -2
- data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +0 -6
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +5 -4
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb +0 -6
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb +0 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +8 -8
- data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +5 -5
- data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +33 -13
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +52 -28
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +2 -1
- data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +6 -5
- data/lib/karafka/pro/scheduled_messages/consumer.rb +7 -1
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +6 -1
- data/lib/karafka/pro/swarm/liveness_listener.rb +118 -13
- data/lib/karafka/pro.rb +0 -6
- data/lib/karafka/processing/consumer_groups/coordinator.rb +4 -5
- data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +6 -2
- data/lib/karafka/processing/consumer_groups/executor.rb +4 -4
- data/lib/karafka/processing/consumer_groups/executors_buffer.rb +2 -4
- data/lib/karafka/processing/consumer_groups/expansions_selector.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +0 -2
- data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +14 -16
- data/lib/karafka/processing/consumer_groups/jobs/consume.rb +2 -5
- data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +2 -3
- data/lib/karafka/processing/consumer_groups/jobs/idle.rb +2 -2
- data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +2 -4
- data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +2 -2
- data/lib/karafka/processing/consumer_groups/partitioner.rb +3 -6
- data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +2 -4
- data/lib/karafka/processing/consumer_groups/strategies/base.rb +3 -5
- data/lib/karafka/processing/consumer_groups/strategies/default.rb +43 -13
- data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +5 -2
- data/lib/karafka/processing/consumer_groups/strategies/mom.rb +2 -3
- data/lib/karafka/processing/consumer_groups/strategy_selector.rb +0 -2
- data/lib/karafka/processing/coordinator.rb +0 -2
- data/lib/karafka/processing/coordinators_buffer.rb +0 -2
- data/lib/karafka/processing/partitioner.rb +0 -2
- data/lib/karafka/processing/strategy_selector.rb +0 -2
- data/lib/karafka/processing/worker.rb +14 -6
- data/lib/karafka/processing/workers_pool.rb +21 -4
- data/lib/karafka/railtie.rb +0 -1
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +6 -5
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/topic.rb +0 -1
- data/lib/karafka/routing/subscription_group.rb +1 -1
- data/lib/karafka/routing/topic.rb +4 -4
- data/lib/karafka/routing/topics.rb +21 -1
- data/lib/karafka/server.rb +9 -0
- data/lib/karafka/setup/config.rb +54 -10
- data/lib/karafka/setup/config_proxy.rb +2 -1
- data/lib/karafka/setup/contracts/config.rb +14 -0
- data/lib/karafka/swarm/node.rb +2 -2
- data/lib/karafka/version.rb +1 -2
- data/lib/karafka.rb +2 -2
- metadata +12 -4
|
@@ -31,9 +31,7 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Pro consumer-group-specific processing components
|
|
35
34
|
module ConsumerGroups
|
|
36
|
-
# Pro jobs
|
|
37
35
|
module Jobs
|
|
38
36
|
# Non-Blocking version of the Eofed job
|
|
39
37
|
# We use this version for LRJ topics for cases where saturated resources would not allow
|
|
@@ -31,7 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Pro consumer-group-specific processing components
|
|
35
34
|
module ConsumerGroups
|
|
36
35
|
# Pro jobs
|
|
37
36
|
module Jobs
|
|
@@ -40,8 +39,8 @@ module Karafka
|
|
|
40
39
|
class Periodic < Karafka::Processing::Jobs::Base
|
|
41
40
|
self.action = :tick
|
|
42
41
|
|
|
43
|
-
# @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor] pro executor that
|
|
44
|
-
# run a given job
|
|
42
|
+
# @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor] pro executor that
|
|
43
|
+
# is suppose to run a given job
|
|
45
44
|
def initialize(executor)
|
|
46
45
|
@executor = executor
|
|
47
46
|
super()
|
|
@@ -31,9 +31,7 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Pro consumer-group-specific processing components
|
|
35
34
|
module ConsumerGroups
|
|
36
|
-
# Pro jobs
|
|
37
35
|
module Jobs
|
|
38
36
|
# Non-Blocking version of the Periodic job
|
|
39
37
|
# We use this version for LRJ topics for cases where saturated resources would not allow
|
|
@@ -30,11 +30,8 @@
|
|
|
30
30
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
|
-
# Pro components related to processing part of Karafka
|
|
34
33
|
module Processing
|
|
35
|
-
# Pro consumer-group-specific processing components
|
|
36
34
|
module ConsumerGroups
|
|
37
|
-
# Pro jobs
|
|
38
35
|
module Jobs
|
|
39
36
|
# The revoked job type in a non-blocking variant.
|
|
40
37
|
# This variant works "like" the regular revoked but does not block the queue.
|
|
@@ -42,8 +42,8 @@ module Karafka
|
|
|
42
42
|
# @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
|
|
43
43
|
# @param messages [Karafka::Messages::Messages] messages batch to be consumed
|
|
44
44
|
# @return [Karafka::Processing::ConsumerGroups::Jobs::Consume] blocking job
|
|
45
|
-
# @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::ConsumeNonBlocking] non
|
|
46
|
-
# for lrj
|
|
45
|
+
# @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::ConsumeNonBlocking] non
|
|
46
|
+
# blocking for lrj
|
|
47
47
|
def consume(executor, messages)
|
|
48
48
|
if executor.topic.long_running_job?
|
|
49
49
|
Jobs::ConsumeNonBlocking.new(executor, messages)
|
|
@@ -54,8 +54,9 @@ module Karafka
|
|
|
54
54
|
|
|
55
55
|
# @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
|
|
56
56
|
# @return [Karafka::Processing::ConsumerGroups::Jobs::Eofed] eofed job for non LRJ
|
|
57
|
-
# @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::EofedNonBlocking] eofed job
|
|
58
|
-
# is non-blocking, so when revocation job is scheduled for LRJ it also will not
|
|
57
|
+
# @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::EofedNonBlocking] eofed job
|
|
58
|
+
# that is non-blocking, so when revocation job is scheduled for LRJ it also will not
|
|
59
|
+
# block
|
|
59
60
|
def eofed(executor)
|
|
60
61
|
if executor.topic.long_running_job?
|
|
61
62
|
Jobs::EofedNonBlocking.new(executor)
|
|
@@ -31,16 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
|
-
# Offset Metadata support on the processing side
|
|
39
35
|
module OffsetMetadata
|
|
40
36
|
# Extra API methods for offset metadata fetching
|
|
41
|
-
# @note Part of this feature API is embedded directly into the strategies because it
|
|
42
|
-
# how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate.
|
|
43
|
-
# of that, they had to be embedded into the strategies.
|
|
37
|
+
# @note Part of this feature API is embedded directly into the strategies because it
|
|
38
|
+
# alters how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate.
|
|
39
|
+
# Because of that, they had to be embedded into the strategies.
|
|
44
40
|
module Consumer
|
|
45
41
|
# @param cache [Boolean] should we use cached result if present (true by default)
|
|
46
42
|
# @return [false, Object] false in case we do not own the partition anymore or
|
|
@@ -31,19 +31,16 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Offset Metadata support on the processing side
|
|
39
36
|
module OffsetMetadata
|
|
40
37
|
# This fetcher is responsible for fetching and caching committed offsets metadata
|
|
41
38
|
# information.
|
|
42
39
|
#
|
|
43
|
-
# By design we fetch all information for a requested topic assignments. Not all topics
|
|
44
|
-
# the same subscription group may need metadata and even if, we can run the few
|
|
45
|
-
# queries. This approach prevents us from querying all assigned topics data in one
|
|
46
|
-
# preventing excessive queries.
|
|
40
|
+
# By design we fetch all information for a requested topic assignments. Not all topics
|
|
41
|
+
# from the same subscription group may need metadata and even if, we can run the few
|
|
42
|
+
# smaller queries. This approach prevents us from querying all assigned topics data in one
|
|
43
|
+
# go preventing excessive queries.
|
|
47
44
|
#
|
|
48
45
|
# Since the assumption is, that user will not have to reach out for the later metadata
|
|
49
46
|
# since it is produced in the context of a given consumer assignment, we can cache the
|
|
@@ -64,7 +61,8 @@ module Karafka
|
|
|
64
61
|
@tpls = {}
|
|
65
62
|
end
|
|
66
63
|
|
|
67
|
-
# Registers a client of a given subscription group, so we can use it for queries later
|
|
64
|
+
# Registers a client of a given subscription group, so we can use it for queries later
|
|
65
|
+
# on
|
|
68
66
|
# @param client [Karafka::Connection::Client]
|
|
69
67
|
# @note Since we store the client reference and not the underlying rdkafka consumer
|
|
70
68
|
# instance, we do not have to deal with the recovery as it is abstracted away
|
|
@@ -104,8 +102,8 @@ module Karafka
|
|
|
104
102
|
|
|
105
103
|
# Clears cache of a given subscription group. It is triggered on assignment changes.
|
|
106
104
|
#
|
|
107
|
-
# @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
|
|
108
|
-
# we want to clear.
|
|
105
|
+
# @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
|
|
106
|
+
# that we want to clear.
|
|
109
107
|
def clear(subscription_group)
|
|
110
108
|
@mutexes.fetch(subscription_group).synchronize do
|
|
111
109
|
@tpls[subscription_group].clear
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module OffsetMetadata
|
|
39
36
|
# Keeps track of rebalances and updates the fetcher
|
|
@@ -31,15 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module ParallelSegments
|
|
39
36
|
# Module for filters injected into the processing pipeline of each of the topics used
|
|
40
37
|
# within the parallel segmented consumer groups
|
|
41
38
|
module Filters
|
|
42
|
-
# Base class for filters for parallel segments that deal with different feature
|
|
39
|
+
# Base class for filters for parallel segments that deal with different feature
|
|
40
|
+
# scenarios
|
|
43
41
|
class Base < Pro::Processing::ConsumerGroups::Filters::Base
|
|
44
42
|
# @param segment_id [Integer] numeric id of the parallel segment group to use with the
|
|
45
43
|
# partitioner and reducer for segment matching comparison
|
|
@@ -82,7 +80,7 @@ module Karafka
|
|
|
82
80
|
|
|
83
81
|
@reducer.call(message_segment_key)
|
|
84
82
|
rescue => e
|
|
85
|
-
#
|
|
83
|
+
# See `#partition` method error handling doc
|
|
86
84
|
Karafka.monitor.instrument(
|
|
87
85
|
"error.occurred",
|
|
88
86
|
caller: self,
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Processing components namespace for parallel segments feature
|
|
39
36
|
module ParallelSegments
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module ParallelSegments
|
|
39
36
|
module Filters
|
|
@@ -31,16 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Pro partitioner that can distribute work based on the virtual partitioner settings
|
|
39
36
|
class Partitioner < Karafka::Processing::ConsumerGroups::Partitioner
|
|
40
37
|
# @param topic [String] topic name
|
|
41
38
|
# @param messages [Array<Karafka::Messages::Message>] karafka messages
|
|
42
|
-
# @param coordinator [Karafka::Pro::Processing::ConsumerGroups::Coordinator]
|
|
43
|
-
# will be used with those messages
|
|
39
|
+
# @param coordinator [Karafka::Pro::Processing::ConsumerGroups::Coordinator]
|
|
40
|
+
# processing coordinator that will be used with those messages
|
|
44
41
|
# @yieldparam [Integer] group id
|
|
45
42
|
# @yieldparam [Array<Karafka::Messages::Message>] karafka messages
|
|
46
43
|
def call(topic, messages, coordinator)
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# Namespace for periodic jobs related processing APIs
|
|
39
36
|
module PeriodicJob
|
|
@@ -64,8 +61,8 @@ module Karafka
|
|
|
64
61
|
end
|
|
65
62
|
|
|
66
63
|
# Runs the on-schedule tick periodic operations
|
|
67
|
-
# This method is an alias but is part of the naming convention used for other flows,
|
|
68
|
-
# is why we do not reference the `handle_before_schedule_tick` directly
|
|
64
|
+
# This method is an alias but is part of the naming convention used for other flows,
|
|
65
|
+
# this is why we do not reference the `handle_before_schedule_tick` directly
|
|
69
66
|
def on_before_schedule_tick
|
|
70
67
|
handle_before_schedule_tick
|
|
71
68
|
end
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
# All code needed for messages piping in Karafka
|
|
39
36
|
module Piping
|
|
@@ -48,8 +45,8 @@ module Karafka
|
|
|
48
45
|
private_constant :EMPTY_HASH
|
|
49
46
|
|
|
50
47
|
# Pipes given message to the provided topic with expected details. Useful for
|
|
51
|
-
# pass-through operations where deserialization is not needed. Upon usage it will
|
|
52
|
-
# all the source headers + meta headers about the source of message.
|
|
48
|
+
# pass-through operations where deserialization is not needed. Upon usage it will
|
|
49
|
+
# include all the source headers + meta headers about the source of message.
|
|
53
50
|
#
|
|
54
51
|
# @param topic [String, Symbol] where we want to send the message
|
|
55
52
|
# @param message [Karafka::Messages::Message] source message to pipe
|
|
@@ -31,17 +31,14 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# DLQ enabled
|
|
42
|
-
# Filtering enabled
|
|
43
|
-
# Long-Running Job enabled
|
|
44
|
-
# Manual offset management enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - DLQ enabled
|
|
39
|
+
# - Filtering enabled
|
|
40
|
+
# - Long-Running Job enabled
|
|
41
|
+
# - Manual offset management enabled
|
|
45
42
|
module DlqFtrLrjMom
|
|
46
43
|
include Strategies::Aj::FtrMom
|
|
47
44
|
include Strategies::Aj::DlqMom
|
|
@@ -67,7 +64,7 @@ module Karafka
|
|
|
67
64
|
|
|
68
65
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
69
66
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
70
|
-
return unless coordinator.filter.
|
|
67
|
+
return unless coordinator.filter.skip?
|
|
71
68
|
elsif !revoked?
|
|
72
69
|
# no need to check for manual seek because AJ consumer is internal and
|
|
73
70
|
# fully controlled by us
|
|
@@ -77,6 +74,10 @@ module Karafka
|
|
|
77
74
|
resume
|
|
78
75
|
else
|
|
79
76
|
apply_dlq_flow do
|
|
77
|
+
# LRJ revocation runs concurrently with the job. If the partition was
|
|
78
|
+
# revoked we must not dispatch: the new owner will reprocess and dispatch
|
|
79
|
+
return resume if revoked?
|
|
80
|
+
|
|
80
81
|
skippable_message, = find_skippable_message
|
|
81
82
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
|
82
83
|
mark_dispatched_to_dlq(skippable_message)
|
|
@@ -31,18 +31,15 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# DLQ enabled
|
|
42
|
-
# Filtering enabled
|
|
43
|
-
# Long-Running Job enabled
|
|
44
|
-
# Manual offset management enabled
|
|
45
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - DLQ enabled
|
|
39
|
+
# - Filtering enabled
|
|
40
|
+
# - Long-Running Job enabled
|
|
41
|
+
# - Manual offset management enabled
|
|
42
|
+
# - Virtual Partitions enabled
|
|
46
43
|
module DlqFtrLrjMomVp
|
|
47
44
|
include Strategies::Aj::FtrMom
|
|
48
45
|
include Strategies::Aj::DlqMomVp
|
|
@@ -73,7 +70,7 @@ module Karafka
|
|
|
73
70
|
|
|
74
71
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
75
72
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
76
|
-
return unless coordinator.filter.
|
|
73
|
+
return unless coordinator.filter.skip?
|
|
77
74
|
elsif !revoked?
|
|
78
75
|
# no need to check for manual seek because AJ consumer is internal and
|
|
79
76
|
# fully controlled by us
|
|
@@ -83,6 +80,10 @@ module Karafka
|
|
|
83
80
|
resume
|
|
84
81
|
else
|
|
85
82
|
apply_dlq_flow do
|
|
83
|
+
# LRJ revocation runs concurrently with the job. If the partition was
|
|
84
|
+
# revoked we must not dispatch: the new owner will reprocess and dispatch
|
|
85
|
+
return resume if revoked?
|
|
86
|
+
|
|
86
87
|
skippable_message, = find_skippable_message
|
|
87
88
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
|
88
89
|
mark_dispatched_to_dlq(skippable_message)
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
@@ -31,9 +31,6 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
@@ -31,16 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# DLQ enabled
|
|
42
|
-
# Long-Running Job enabled
|
|
43
|
-
# Manual offset management enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - DLQ enabled
|
|
39
|
+
# - Long-Running Job enabled
|
|
40
|
+
# - Manual offset management enabled
|
|
44
41
|
#
|
|
45
42
|
# This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
|
|
46
43
|
# offset management works in this case.
|
|
@@ -70,6 +67,10 @@ module Karafka
|
|
|
70
67
|
resume
|
|
71
68
|
else
|
|
72
69
|
apply_dlq_flow do
|
|
70
|
+
# LRJ revocation runs concurrently with the job. If the partition was
|
|
71
|
+
# revoked we must not dispatch: the new owner will reprocess and dispatch
|
|
72
|
+
return resume if revoked?
|
|
73
|
+
|
|
73
74
|
skippable_message, = find_skippable_message
|
|
74
75
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
|
75
76
|
mark_dispatched_to_dlq(skippable_message)
|
|
@@ -31,17 +31,14 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# DLQ enabled
|
|
42
|
-
# Long-Running Job enabled
|
|
43
|
-
# Manual offset management enabled
|
|
44
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - DLQ enabled
|
|
39
|
+
# - Long-Running Job enabled
|
|
40
|
+
# - Manual offset management enabled
|
|
41
|
+
# - Virtual Partitions enabled
|
|
45
42
|
#
|
|
46
43
|
# This case is a bit of special. Please see the `Aj::DlqMom` for explanation on how the
|
|
47
44
|
# offset management works in this case.
|
|
@@ -74,6 +71,10 @@ module Karafka
|
|
|
74
71
|
resume
|
|
75
72
|
else
|
|
76
73
|
apply_dlq_flow do
|
|
74
|
+
# LRJ revocation runs concurrently with the job. If the partition was
|
|
75
|
+
# revoked we must not dispatch: the new owner will reprocess and dispatch
|
|
76
|
+
return resume if revoked?
|
|
77
|
+
|
|
77
78
|
skippable_message, = find_skippable_message
|
|
78
79
|
dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
|
|
79
80
|
mark_dispatched_to_dlq(skippable_message)
|
|
@@ -31,18 +31,15 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# DLQ enabled
|
|
42
|
-
# Manual offset management enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - DLQ enabled
|
|
39
|
+
# - Manual offset management enabled
|
|
43
40
|
#
|
|
44
|
-
# AJ has manual offset management on by default and the offset management is delegated
|
|
45
|
-
# the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
|
|
41
|
+
# AJ has manual offset management on by default and the offset management is delegated
|
|
42
|
+
# to the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
|
|
46
43
|
# consumed when we skip given job upon errors. In all the other scenarios marking as
|
|
47
44
|
# consumed needs to happen in the AJ consumer on a per job basis.
|
|
48
45
|
module DlqMom
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# Manual offset management enabled
|
|
42
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Manual offset management enabled
|
|
39
|
+
# - Virtual Partitions enabled
|
|
43
40
|
module DlqMomVp
|
|
44
41
|
include Strategies::Default
|
|
45
42
|
include Strategies::Dlq::Vp
|
|
@@ -31,16 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# Long-Running Job enabled
|
|
43
|
-
# Manual offset management enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Long-Running Job enabled
|
|
40
|
+
# - Manual offset management enabled
|
|
44
41
|
module FtrLrjMom
|
|
45
42
|
include Strategies::Lrj::FtrMom
|
|
46
43
|
|
|
@@ -31,17 +31,14 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# Filtering enabled
|
|
42
|
-
# Long-Running Job enabled
|
|
43
|
-
# Manual offset management enabled
|
|
44
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Long-Running Job enabled
|
|
40
|
+
# - Manual offset management enabled
|
|
41
|
+
# - Virtual Partitions enabled
|
|
45
42
|
module FtrLrjMomVp
|
|
46
43
|
include Strategies::Vp::Default
|
|
47
44
|
include Strategies::Lrj::FtrMom
|
|
@@ -69,7 +66,7 @@ module Karafka
|
|
|
69
66
|
|
|
70
67
|
# :seek and :pause are fully handled by handle_post_filtering
|
|
71
68
|
# For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
|
|
72
|
-
return unless coordinator.filter.
|
|
69
|
+
return unless coordinator.filter.skip?
|
|
73
70
|
elsif !revoked?
|
|
74
71
|
# no need to check for manual seek because AJ consumer is internal and
|
|
75
72
|
# fully controlled by us
|
|
@@ -31,16 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
|
-
# Namespace for ActiveJob related strategies
|
|
40
36
|
module Aj
|
|
41
|
-
# ActiveJob enabled
|
|
42
|
-
# Filtering enabled
|
|
43
|
-
# Manual Offset management enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Manual Offset management enabled
|
|
44
40
|
module FtrMom
|
|
45
41
|
# Same as standard Mom::Ftr
|
|
46
42
|
include Strategies::Mom::Ftr
|
|
@@ -31,17 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
|
-
# Namespace for ActiveJob related strategies
|
|
40
36
|
module Aj
|
|
41
|
-
# ActiveJob enabled
|
|
42
|
-
# Filtering enabled
|
|
43
|
-
# Manual Offset management enabled
|
|
44
|
-
# Virtual partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Filtering enabled
|
|
39
|
+
# - Manual Offset management enabled
|
|
40
|
+
# - Virtual partitions enabled
|
|
45
41
|
module FtrMomVp
|
|
46
42
|
include Strategies::Aj::FtrMom
|
|
47
43
|
include Strategies::Aj::MomVp
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# Long-Running Job enabled
|
|
42
|
-
# Manual offset management enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Long-Running Job enabled
|
|
39
|
+
# - Manual offset management enabled
|
|
43
40
|
module LrjMom
|
|
44
41
|
# Same behaviour as Lrj::Mom
|
|
45
42
|
include Strategies::Lrj::Mom
|
|
@@ -31,16 +31,13 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
module Aj
|
|
40
|
-
# ActiveJob enabled
|
|
41
|
-
# Long-Running Job enabled
|
|
42
|
-
# Manual offset management enabled
|
|
43
|
-
# Virtual Partitions enabled
|
|
37
|
+
# - ActiveJob enabled
|
|
38
|
+
# - Long-Running Job enabled
|
|
39
|
+
# - Manual offset management enabled
|
|
40
|
+
# - Virtual Partitions enabled
|
|
44
41
|
module LrjMomVp
|
|
45
42
|
include Strategies::Default
|
|
46
43
|
include Strategies::Vp::Default
|
|
@@ -31,15 +31,12 @@
|
|
|
31
31
|
module Karafka
|
|
32
32
|
module Pro
|
|
33
33
|
module Processing
|
|
34
|
-
# Consumer-group-specific Pro processing components (driven by rebalance callbacks and
|
|
35
|
-
# partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
|
|
36
|
-
# lands.
|
|
37
34
|
module ConsumerGroups
|
|
38
35
|
module Strategies
|
|
39
36
|
# Namespace for ActiveJob related strategies
|
|
40
37
|
module Aj
|
|
41
|
-
# ActiveJob enabled
|
|
42
|
-
# Manual Offset management enabled
|
|
38
|
+
# - ActiveJob enabled
|
|
39
|
+
# - Manual Offset management enabled
|
|
43
40
|
module Mom
|
|
44
41
|
# Standard ActiveJob strategy is the same one we use for Mom
|
|
45
42
|
include Strategies::Mom::Default
|