karafka 2.4.15 → 2.4.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +5 -5
- data/.ruby-version +1 -1
- data/CHANGELOG.md +15 -0
- data/Gemfile +1 -0
- data/Gemfile.lock +17 -14
- data/LICENSE +7 -2
- data/config/locales/errors.yml +1 -0
- data/docker-compose.yml +0 -2
- data/lib/karafka/base_consumer.rb +70 -16
- data/lib/karafka/connection/client.rb +2 -16
- data/lib/karafka/connection/rebalance_manager.rb +24 -13
- data/lib/karafka/instrumentation/logger_listener.rb +3 -0
- data/lib/karafka/instrumentation/notifications.rb +4 -0
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +1 -26
- data/lib/karafka/pro/active_job/consumer.rb +2 -10
- data/lib/karafka/pro/active_job/dispatcher.rb +2 -10
- data/lib/karafka/pro/active_job/job_options_contract.rb +2 -10
- data/lib/karafka/pro/base_consumer.rb +2 -10
- data/lib/karafka/pro/cleaner/errors.rb +2 -10
- data/lib/karafka/pro/cleaner/messages/message.rb +14 -12
- data/lib/karafka/pro/cleaner/messages/messages.rb +2 -10
- data/lib/karafka/pro/cleaner/messages/metadata.rb +41 -0
- data/lib/karafka/pro/cleaner.rb +3 -10
- data/lib/karafka/pro/connection/manager.rb +2 -10
- data/lib/karafka/pro/connection/multiplexing/listener.rb +2 -10
- data/lib/karafka/pro/contracts/base.rb +2 -10
- data/lib/karafka/pro/contracts/server_cli_options.rb +2 -10
- data/lib/karafka/pro/encryption/cipher.rb +2 -10
- data/lib/karafka/pro/encryption/contracts/config.rb +2 -10
- data/lib/karafka/pro/encryption/errors.rb +2 -10
- data/lib/karafka/pro/encryption/messages/middleware.rb +2 -10
- data/lib/karafka/pro/encryption/messages/parser.rb +2 -10
- data/lib/karafka/pro/encryption/setup/config.rb +2 -10
- data/lib/karafka/pro/encryption.rb +2 -10
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +2 -10
- data/lib/karafka/pro/iterator/expander.rb +2 -10
- data/lib/karafka/pro/iterator/tpl_builder.rb +2 -10
- data/lib/karafka/pro/iterator.rb +2 -10
- data/lib/karafka/pro/loader.rb +2 -10
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +2 -10
- data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +2 -10
- data/lib/karafka/pro/processing/collapser.rb +2 -10
- data/lib/karafka/pro/processing/coordinator.rb +2 -10
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +2 -10
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +2 -10
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +2 -10
- data/lib/karafka/pro/processing/executor.rb +2 -10
- data/lib/karafka/pro/processing/expansions_selector.rb +2 -10
- data/lib/karafka/pro/processing/filters/base.rb +2 -10
- data/lib/karafka/pro/processing/filters/delayer.rb +2 -10
- data/lib/karafka/pro/processing/filters/expirer.rb +2 -10
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +2 -10
- data/lib/karafka/pro/processing/filters/throttler.rb +2 -10
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +2 -10
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +4 -10
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +4 -10
- data/lib/karafka/pro/processing/jobs/periodic.rb +4 -10
- data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +4 -10
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +4 -10
- data/lib/karafka/pro/processing/jobs_builder.rb +2 -10
- data/lib/karafka/pro/processing/jobs_queue.rb +2 -10
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +2 -10
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +2 -10
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +2 -10
- data/lib/karafka/pro/processing/partitioner.rb +2 -10
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +2 -10
- data/lib/karafka/pro/processing/piping/consumer.rb +2 -10
- data/lib/karafka/pro/processing/schedulers/base.rb +2 -10
- data/lib/karafka/pro/processing/schedulers/default.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +3 -11
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +3 -11
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +3 -11
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +3 -11
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +3 -11
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +5 -13
- data/lib/karafka/pro/processing/strategies/aj/mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/base.rb +2 -10
- data/lib/karafka/pro/processing/strategies/default.rb +135 -77
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +6 -14
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +3 -11
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +3 -11
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +3 -11
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +3 -11
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +3 -11
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +3 -11
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +4 -12
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +5 -13
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +3 -11
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +2 -10
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +4 -12
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/lrj/vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/mom/default.rb +2 -10
- data/lib/karafka/pro/processing/strategies/mom/ftr.rb +2 -10
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/mom/vp.rb +2 -10
- data/lib/karafka/pro/processing/strategies/vp/default.rb +5 -10
- data/lib/karafka/pro/processing/strategies.rb +2 -10
- data/lib/karafka/pro/processing/strategy_selector.rb +2 -10
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/consumer.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/errors.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/executor.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/listener.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/matcher.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/schedule.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/serializer.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +2 -10
- data/lib/karafka/pro/recurring_tasks/task.rb +2 -10
- data/lib/karafka/pro/recurring_tasks.rb +2 -10
- data/lib/karafka/pro/routing/features/active_job/builder.rb +2 -10
- data/lib/karafka/pro/routing/features/active_job.rb +2 -10
- data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +2 -10
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/adaptive_iterator.rb +2 -10
- data/lib/karafka/pro/routing/features/base.rb +2 -10
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +2 -10
- data/lib/karafka/pro/routing/features/delaying/config.rb +2 -10
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/delaying/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/delaying.rb +2 -10
- data/lib/karafka/pro/routing/features/direct_assignments/config.rb +2 -10
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +2 -10
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +2 -10
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/direct_assignments.rb +2 -10
- data/lib/karafka/pro/routing/features/expiring/config.rb +2 -10
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/expiring/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/expiring.rb +2 -10
- data/lib/karafka/pro/routing/features/filtering/config.rb +2 -10
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/filtering/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/filtering.rb +2 -10
- data/lib/karafka/pro/routing/features/inline_insights/config.rb +2 -10
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/inline_insights/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/inline_insights.rb +2 -10
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +2 -10
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/long_running_job.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing/config.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +2 -10
- data/lib/karafka/pro/routing/features/multiplexing.rb +2 -10
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/non_blocking_job.rb +2 -10
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +2 -10
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/offset_metadata.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/builder.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/config.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/detector.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns/topics.rb +2 -10
- data/lib/karafka/pro/routing/features/patterns.rb +2 -10
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/pausing/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/pausing.rb +2 -10
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +2 -10
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/periodic_job.rb +2 -10
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +2 -10
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +2 -10
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +2 -10
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +2 -10
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +2 -10
- data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +2 -10
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +2 -10
- data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/scheduled_messages.rb +2 -10
- data/lib/karafka/pro/routing/features/swarm/config.rb +2 -10
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +2 -10
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/swarm/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/swarm.rb +2 -10
- data/lib/karafka/pro/routing/features/throttling/config.rb +2 -10
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/throttling/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/throttling.rb +2 -10
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +2 -10
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +2 -10
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/consumer.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/day.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/errors.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/proxy.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/serializer.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/state.rb +2 -10
- data/lib/karafka/pro/scheduled_messages/tracker.rb +2 -10
- data/lib/karafka/pro/scheduled_messages.rb +2 -10
- data/lib/karafka/pro/swarm/liveness_listener.rb +2 -10
- data/lib/karafka/processing/executor.rb +7 -0
- data/lib/karafka/processing/jobs/base.rb +13 -0
- data/lib/karafka/processing/jobs/consume.rb +2 -0
- data/lib/karafka/processing/jobs/eofed.rb +2 -0
- data/lib/karafka/processing/jobs/idle.rb +2 -0
- data/lib/karafka/processing/jobs/revoked.rb +2 -0
- data/lib/karafka/processing/jobs/shutdown.rb +2 -0
- data/lib/karafka/processing/strategies/aj_dlq_mom.rb +1 -1
- data/lib/karafka/processing/strategies/default.rb +28 -17
- data/lib/karafka/processing/strategies/dlq.rb +4 -4
- data/lib/karafka/processing/strategies/dlq_mom.rb +2 -2
- data/lib/karafka/processing/worker.rb +21 -18
- data/lib/karafka/version.rb +1 -1
- data.tar.gz.sig +0 -0
- metadata +4 -6
- metadata.gz.sig +0 -0
@@ -24,7 +24,7 @@ module Karafka
|
|
24
24
|
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
25
25
|
# No actions needed for the standard flow here
|
26
26
|
def handle_before_schedule_#{action}
|
27
|
-
|
27
|
+
monitor.instrument('consumer.before_schedule_#{action}', caller: self)
|
28
28
|
|
29
29
|
nil
|
30
30
|
end
|
@@ -35,8 +35,8 @@ module Karafka
|
|
35
35
|
# @note It runs in the listener loop. Should **not** be used for anything heavy or
|
36
36
|
# with any potential errors. Mostly for initialization of states, etc.
|
37
37
|
def handle_initialized
|
38
|
-
|
39
|
-
|
38
|
+
monitor.instrument('consumer.initialize', caller: self)
|
39
|
+
monitor.instrument('consumer.initialized', caller: self) do
|
40
40
|
initialized
|
41
41
|
end
|
42
42
|
end
|
@@ -54,13 +54,13 @@ module Karafka
|
|
54
54
|
def mark_as_consumed(message)
|
55
55
|
# seek offset can be nil only in case `#seek` was invoked with offset reset request
|
56
56
|
# In case like this we ignore marking
|
57
|
-
return true if
|
57
|
+
return true if seek_offset.nil?
|
58
58
|
# Ignore earlier offsets than the one we already committed
|
59
|
-
return true if
|
59
|
+
return true if seek_offset > message.offset
|
60
60
|
return false if revoked?
|
61
61
|
return revoked? unless client.mark_as_consumed(message)
|
62
62
|
|
63
|
-
|
63
|
+
self.seek_offset = message.offset + 1
|
64
64
|
|
65
65
|
true
|
66
66
|
end
|
@@ -73,14 +73,14 @@ module Karafka
|
|
73
73
|
def mark_as_consumed!(message)
|
74
74
|
# seek offset can be nil only in case `#seek` was invoked with offset reset request
|
75
75
|
# In case like this we ignore marking
|
76
|
-
return true if
|
76
|
+
return true if seek_offset.nil?
|
77
77
|
# Ignore earlier offsets than the one we already committed
|
78
|
-
return true if
|
78
|
+
return true if seek_offset > message.offset
|
79
79
|
return false if revoked?
|
80
80
|
|
81
81
|
return revoked? unless client.mark_as_consumed!(message)
|
82
82
|
|
83
|
-
|
83
|
+
self.seek_offset = message.offset + 1
|
84
84
|
|
85
85
|
true
|
86
86
|
end
|
@@ -115,10 +115,21 @@ module Karafka
|
|
115
115
|
coordinator.pause_tracker.increment
|
116
116
|
end
|
117
117
|
|
118
|
+
# Runs the wrapping to execute appropriate action wrapped with the wrapper method code
|
119
|
+
#
|
120
|
+
# @param action [Symbol]
|
121
|
+
# @param block [Proc]
|
122
|
+
def handle_wrap(action, &block)
|
123
|
+
monitor.instrument('consumer.wrap', caller: self)
|
124
|
+
monitor.instrument('consumer.wrapped', caller: self) do
|
125
|
+
wrap(action, &block)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
118
129
|
# Run the user consumption code
|
119
130
|
def handle_consume
|
120
|
-
|
121
|
-
|
131
|
+
monitor.instrument('consumer.consume', caller: self)
|
132
|
+
monitor.instrument('consumer.consumed', caller: self) do
|
122
133
|
consume
|
123
134
|
end
|
124
135
|
|
@@ -164,8 +175,8 @@ module Karafka
|
|
164
175
|
|
165
176
|
# Runs the consumer `#eofed` method with reporting
|
166
177
|
def handle_eofed
|
167
|
-
|
168
|
-
|
178
|
+
monitor.instrument('consumer.eof', caller: self)
|
179
|
+
monitor.instrument('consumer.eofed', caller: self) do
|
169
180
|
eofed
|
170
181
|
end
|
171
182
|
ensure
|
@@ -180,8 +191,8 @@ module Karafka
|
|
180
191
|
|
181
192
|
coordinator.revoke
|
182
193
|
|
183
|
-
|
184
|
-
|
194
|
+
monitor.instrument('consumer.revoke', caller: self)
|
195
|
+
monitor.instrument('consumer.revoked', caller: self) do
|
185
196
|
revoked
|
186
197
|
end
|
187
198
|
ensure
|
@@ -190,8 +201,8 @@ module Karafka
|
|
190
201
|
|
191
202
|
# Runs the shutdown code
|
192
203
|
def handle_shutdown
|
193
|
-
|
194
|
-
|
204
|
+
monitor.instrument('consumer.shutting_down', caller: self)
|
205
|
+
monitor.instrument('consumer.shutdown', caller: self) do
|
195
206
|
shutdown
|
196
207
|
end
|
197
208
|
ensure
|
@@ -81,11 +81,11 @@ module Karafka
|
|
81
81
|
|
82
82
|
return if revoked?
|
83
83
|
else
|
84
|
-
|
84
|
+
self.seek_offset = skippable_message.offset + 1
|
85
85
|
end
|
86
86
|
|
87
87
|
# We pause to backoff once just in case.
|
88
|
-
pause(
|
88
|
+
pause(seek_offset, nil, false)
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
@@ -95,7 +95,7 @@ module Karafka
|
|
95
95
|
# information if this message was from marked offset or figured out via mom flow
|
96
96
|
def find_skippable_message
|
97
97
|
skippable_message = messages.find do |msg|
|
98
|
-
coordinator.marked? && msg.offset ==
|
98
|
+
coordinator.marked? && msg.offset == seek_offset
|
99
99
|
end
|
100
100
|
|
101
101
|
# If we don't have the message matching the last comitted offset, it means that
|
@@ -117,7 +117,7 @@ module Karafka
|
|
117
117
|
)
|
118
118
|
|
119
119
|
# Notify about dispatch on the events bus
|
120
|
-
|
120
|
+
monitor.instrument(
|
121
121
|
'dead_letter_queue.dispatched',
|
122
122
|
caller: self,
|
123
123
|
message: skippable_message
|
@@ -44,10 +44,10 @@ module Karafka
|
|
44
44
|
# in an infinite loop trying to un-pause from the message we've already processed
|
45
45
|
# Of course, since it's a MoM a rebalance or kill, will move it back as no
|
46
46
|
# offsets are being committed
|
47
|
-
|
47
|
+
self.seek_offset = skippable_message.offset + 1
|
48
48
|
end
|
49
49
|
|
50
|
-
pause(
|
50
|
+
pause(seek_offset, nil, false)
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
@@ -19,7 +19,8 @@ module Karafka
|
|
19
19
|
class Worker
|
20
20
|
include Helpers::Async
|
21
21
|
include Helpers::ConfigImporter.new(
|
22
|
-
worker_job_call_wrapper: %i[internal processing worker_job_call_wrapper]
|
22
|
+
worker_job_call_wrapper: %i[internal processing worker_job_call_wrapper],
|
23
|
+
monitor: %i[monitor]
|
23
24
|
)
|
24
25
|
|
25
26
|
# @return [String] id of this worker
|
@@ -53,28 +54,30 @@ module Karafka
|
|
53
54
|
instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
|
54
55
|
|
55
56
|
if job
|
56
|
-
|
57
|
+
job.wrap do
|
58
|
+
monitor.instrument('worker.process', instrument_details)
|
57
59
|
|
58
|
-
|
59
|
-
|
60
|
+
monitor.instrument('worker.processed', instrument_details) do
|
61
|
+
job.before_call
|
60
62
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
63
|
+
# If a job is marked as non blocking, we can run a tick in the job queue and if there
|
64
|
+
# are no other blocking factors, the job queue will be unlocked.
|
65
|
+
# If this does not run, all the things will be blocking and job queue won't allow to
|
66
|
+
# pass it until done.
|
67
|
+
@jobs_queue.tick(job.group_id) if job.non_blocking?
|
66
68
|
|
67
|
-
|
68
|
-
job.call
|
69
|
-
else
|
70
|
-
worker_job_call_wrapper.wrap do
|
69
|
+
if @non_wrapped_flow
|
71
70
|
job.call
|
71
|
+
else
|
72
|
+
worker_job_call_wrapper.wrap do
|
73
|
+
job.call
|
74
|
+
end
|
72
75
|
end
|
73
|
-
end
|
74
76
|
|
75
|
-
|
77
|
+
job.after_call
|
76
78
|
|
77
|
-
|
79
|
+
true
|
80
|
+
end
|
78
81
|
end
|
79
82
|
else
|
80
83
|
false
|
@@ -83,7 +86,7 @@ module Karafka
|
|
83
86
|
# rubocop:disable Lint/RescueException
|
84
87
|
rescue Exception => e
|
85
88
|
# rubocop:enable Lint/RescueException
|
86
|
-
|
89
|
+
monitor.instrument(
|
87
90
|
'error.occurred',
|
88
91
|
caller: self,
|
89
92
|
job: job,
|
@@ -99,7 +102,7 @@ module Karafka
|
|
99
102
|
end
|
100
103
|
|
101
104
|
# Always publish info, that we completed all the work despite its result
|
102
|
-
|
105
|
+
monitor.instrument('worker.completed', instrument_details)
|
103
106
|
end
|
104
107
|
end
|
105
108
|
end
|
data/lib/karafka/version.rb
CHANGED
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,11 +1,10 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.4.
|
4
|
+
version: 2.4.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain:
|
11
10
|
- |
|
@@ -35,7 +34,7 @@ cert_chain:
|
|
35
34
|
i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
|
36
35
|
ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
|
37
36
|
-----END CERTIFICATE-----
|
38
|
-
date:
|
37
|
+
date: 2025-01-15 00:00:00.000000000 Z
|
39
38
|
dependencies:
|
40
39
|
- !ruby/object:Gem::Dependency
|
41
40
|
name: base64
|
@@ -275,6 +274,7 @@ files:
|
|
275
274
|
- lib/karafka/pro/cleaner/errors.rb
|
276
275
|
- lib/karafka/pro/cleaner/messages/message.rb
|
277
276
|
- lib/karafka/pro/cleaner/messages/messages.rb
|
277
|
+
- lib/karafka/pro/cleaner/messages/metadata.rb
|
278
278
|
- lib/karafka/pro/connection/manager.rb
|
279
279
|
- lib/karafka/pro/connection/multiplexing/listener.rb
|
280
280
|
- lib/karafka/pro/contracts/base.rb
|
@@ -605,7 +605,6 @@ metadata:
|
|
605
605
|
source_code_uri: https://github.com/karafka/karafka
|
606
606
|
documentation_uri: https://karafka.io/docs
|
607
607
|
rubygems_mfa_required: 'true'
|
608
|
-
post_install_message:
|
609
608
|
rdoc_options: []
|
610
609
|
require_paths:
|
611
610
|
- lib
|
@@ -620,8 +619,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
620
619
|
- !ruby/object:Gem::Version
|
621
620
|
version: '0'
|
622
621
|
requirements: []
|
623
|
-
rubygems_version: 3.
|
624
|
-
signing_key:
|
622
|
+
rubygems_version: 3.6.2
|
625
623
|
specification_version: 4
|
626
624
|
summary: Karafka is Ruby and Rails efficient Kafka processing framework.
|
627
625
|
test_files: []
|
metadata.gz.sig
CHANGED
Binary file
|