karafka 2.5.5 → 2.5.7

data/lib/karafka/pro/processing/partitioner.rb

@@ -56,16 +56,16 @@ module Karafka
             # user flow but should mitigate damages by not virtualizing
             begin
               groupings = vps.distributor.call(messages)
-            rescue StandardError => e
+            rescue => e
               # This should not happen. If you are seeing this it means your partitioner code
               # failed and raised an error. We highly recommend mitigating partitioner level errors
               # on the user side because this type of collapse should be considered a last resort
               Karafka.monitor.instrument(
-                'error.occurred',
+                "error.occurred",
                 caller: self,
                 error: e,
                 messages: messages,
-                type: 'virtual_partitions.partitioner.error'
+                type: "virtual_partitions.partitioner.error"
               )
 
               groupings = { 0 => messages }

data/lib/karafka/pro/processing/periodic_job/consumer.rb

@@ -40,12 +40,13 @@ module Karafka
               #
               # We need to check both cases (public and private) since user is not expected to
               # have this method public
-              return if consumer_singleton_class.instance_methods.include?(:tick)
+              return if consumer_singleton_class.method_defined?(:tick)
               return if consumer_singleton_class.private_instance_methods.include?(:tick)
 
               # Create empty ticking method
               consumer_singleton_class.class_eval do
-                def tick; end
+                def tick
+                end
               end
             end
           end
@@ -61,12 +62,12 @@ module Karafka
           # @private
           def on_tick
             handle_tick
-          rescue StandardError => e
+          rescue => e
             Karafka.monitor.instrument(
-              'error.occurred',
+              "error.occurred",
               error: e,
               caller: self,
-              type: 'consumer.tick.error'
+              type: "consumer.tick.error"
             )
           end
         end

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -107,20 +107,20 @@ module Karafka
               topic: topic,
               payload: message.raw_payload,
               headers: message.raw_headers.merge(
-                'source_topic' => message.topic,
-                'source_partition' => message.partition.to_s,
-                'source_offset' => message.offset.to_s,
-                'source_consumer_group' => self.topic.consumer_group.id
+                "source_topic" => message.topic,
+                "source_partition" => message.partition.to_s,
+                "source_offset" => message.offset.to_s,
+                "source_consumer_group" => self.topic.consumer_group.id
               )
             }
 
             # Use a key only if key was provided
-            if message.raw_key
-              pipe_message[:key] = message.raw_key
+            pipe_message[:key] = if message.raw_key
+              message.raw_key
             # Otherwise pipe creating a key that will assign it based on the source partition
             # number
             else
-              pipe_message[:key] = message.partition.to_s
+              message.partition.to_s
             end
 
             # Optional method user can define in consumer to enhance the dlq message hash with

data/lib/karafka/pro/processing/schedulers/base.rb

@@ -65,7 +65,7 @@ module Karafka
           #   [Array<Karafka::Processing::Jobs::Consume, Processing::Jobs::ConsumeNonBlocking>]
           #   jobs for scheduling
           def schedule_consumption(_jobs_array)
-            raise NotImplementedError, 'Implement in a subclass'
+            raise NotImplementedError, "Implement in a subclass"
           end
 
           # Runs the revocation jobs scheduling flow under a mutex
@@ -109,10 +109,10 @@ module Karafka
           end
 
           # Schedule by default all except consumption as fifo
-          alias schedule_revocation schedule_fifo
-          alias schedule_shutdown schedule_fifo
-          alias schedule_idle schedule_fifo
-          alias schedule_periodic schedule_fifo
+          alias_method :schedule_revocation, :schedule_fifo
+          alias_method :schedule_shutdown, :schedule_fifo
+          alias_method :schedule_idle, :schedule_fifo
+          alias_method :schedule_periodic, :schedule_fifo
 
           # Runs the manage tick under mutex
           def on_manage

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -71,11 +71,11 @@ module Karafka
           end
 
           # By default all non-consumption work is scheduled in a fifo order
-          alias on_schedule_revocation schedule_fifo
-          alias on_schedule_shutdown schedule_fifo
-          alias on_schedule_idle schedule_fifo
-          alias on_schedule_periodic schedule_fifo
-          alias on_schedule_eofed schedule_fifo
+          alias_method :on_schedule_revocation, :schedule_fifo
+          alias_method :on_schedule_shutdown, :schedule_fifo
+          alias_method :on_schedule_idle, :schedule_fifo
+          alias_method :on_schedule_periodic, :schedule_fifo
+          alias_method :on_schedule_eofed, :schedule_fifo
 
           # This scheduler does not have anything to manage as it is a pass through and has no
           # state

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -52,14 +52,17 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
                     seek(seek_offset, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   apply_dlq_flow do
                     skippable_message, = find_skippable_message

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -58,14 +58,17 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
                     seek(seek_offset, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   apply_dlq_flow do
                     skippable_message, = find_skippable_message

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -54,14 +54,17 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
                     seek(seek_offset, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   retry_after_pause
                 end

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -78,8 +78,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              monitor.instrument('consumer.revoke', caller: self)
-              monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument("consumer.revoke", caller: self)
+              monitor.instrument("consumer.revoked", caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/processing/strategies/default.rb

@@ -82,12 +82,12 @@ module Karafka
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
-                         mark_with_transaction(message, offset_metadata, true)
-                       elsif @_transactional_marking
-                         raise Errors::NonTransactionalMarkingAttemptError
-                       else
-                         client.mark_as_consumed(message, offset_metadata)
-                       end
+                mark_with_transaction(message, offset_metadata, true)
+              elsif @_transactional_marking
+                raise Errors::NonTransactionalMarkingAttemptError
+              else
+                client.mark_as_consumed(message, offset_metadata)
+              end
 
               return revoked? unless stored
 
@@ -124,12 +124,12 @@ module Karafka
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
-                         mark_with_transaction(message, offset_metadata, false)
-                       elsif @_transactional_marking
-                         raise Errors::NonTransactionalMarkingAttemptError
-                       else
-                         client.mark_as_consumed!(message, offset_metadata)
-                       end
+                mark_with_transaction(message, offset_metadata, false)
+              elsif @_transactional_marking
+                raise Errors::NonTransactionalMarkingAttemptError
+              else
+                client.mark_as_consumed!(message, offset_metadata)
+              end
 
               return revoked? unless stored
 
@@ -165,7 +165,7 @@ module Karafka
             default_producer = nil
             transaction_started = nil
 
-            monitor.instrument('consumer.consuming.transaction', caller: self) do
+            monitor.instrument("consumer.consuming.transaction", caller: self) do
               default_producer = producer
               self.producer = active_producer
 
@@ -311,7 +311,7 @@ module Karafka
 
           # No actions needed for the standard flow here
           def handle_before_schedule_consume
-            monitor.instrument('consumer.before_schedule_consume', caller: self)
+            monitor.instrument("consumer.before_schedule_consume", caller: self)
 
             nil
           end
@@ -330,15 +330,15 @@ module Karafka
             # This can happen primarily when an LRJ job gets to the internal worker queue and
             # this partition is revoked prior processing.
             unless revoked?
-              monitor.instrument('consumer.consume', caller: self)
-              monitor.instrument('consumer.consumed', caller: self) do
+              monitor.instrument("consumer.consume", caller: self)
+              monitor.instrument("consumer.consumed", caller: self) do
                 consume
               end
             end
 
             # Mark job as successful
             coordinator.success!(self)
-          rescue StandardError => e
+          rescue => e
             # If failed, mark as failed
             coordinator.failure!(self, e)
 
@@ -377,8 +377,8 @@ module Karafka
               coordinator.revoke
             end
 
-            monitor.instrument('consumer.revoke', caller: self)
-            monitor.instrument('consumer.revoked', caller: self) do
+            monitor.instrument("consumer.revoke", caller: self)
+            monitor.instrument("consumer.revoked", caller: self) do
               revoked
             end
           ensure
@@ -387,15 +387,15 @@ module Karafka
 
           # No action needed for the tick standard flow
           def handle_before_schedule_tick
-            monitor.instrument('consumer.before_schedule_tick', caller: self)
+            monitor.instrument("consumer.before_schedule_tick", caller: self)
 
             nil
           end
 
           # Runs the consumer `#tick` method with reporting
           def handle_tick
-            monitor.instrument('consumer.tick', caller: self)
-            monitor.instrument('consumer.ticked', caller: self) do
+            monitor.instrument("consumer.tick", caller: self)
+            monitor.instrument("consumer.ticked", caller: self) do
               tick
             end
           ensure

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -129,7 +129,7 @@ module Karafka
 
               # Notify about dispatch on the events bus
               monitor.instrument(
-                'dead_letter_queue.dispatched',
+                "dead_letter_queue.dispatched",
                 caller: self,
                 message: skippable_message
               )
@@ -170,12 +170,12 @@ module Karafka
                 partition_key: source_partition,
                 payload: skippable_message.raw_payload,
                 headers: skippable_message.raw_headers.merge(
-                  'source_topic' => topic.name,
-                  'source_partition' => source_partition,
-                  'source_offset' => skippable_message.offset.to_s,
-                  'source_consumer_group' => topic.consumer_group.id,
-                  'source_attempts' => attempt.to_s,
-                  'source_trace_id' => errors_tracker.trace_id
+                  "source_topic" => topic.name,
+                  "source_partition" => source_partition,
+                  "source_offset" => skippable_message.offset.to_s,
+                  "source_consumer_group" => topic.consumer_group.id,
+                  "source_attempts" => attempt.to_s,
+                  "source_trace_id" => errors_tracker.trace_id
                 )
               }
 

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -56,12 +56,15 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked? && !coordinator.manual_seek?
                     seek(seek_offset, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   apply_dlq_flow do
                     return resume if revoked?

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -51,12 +51,15 @@ module Karafka
 
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked? && !coordinator.manual_seek?
                     seek(last_group_message.offset + 1, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   apply_dlq_flow do
                     return resume if revoked?

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -99,7 +99,7 @@ module Karafka
                 throttle_message = filter.cursor
 
                 monitor.instrument(
-                  'filtering.seek',
+                  "filtering.seek",
                   caller: self,
                   message: throttle_message
                 ) do
@@ -114,7 +114,7 @@ module Karafka
                 throttle_message = filter.cursor
 
                 monitor.instrument(
-                  'filtering.throttled',
+                  "filtering.throttled",
                   caller: self,
                   message: throttle_message,
                   timeout: throttle_timeout

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -84,8 +84,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              monitor.instrument('consumer.revoke', caller: self)
-              monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument("consumer.revoke", caller: self)
+              monitor.instrument("consumer.revoked", caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -54,14 +54,17 @@ module Karafka
                   # If still not revoked and was throttled, we need to apply throttling logic
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
                     seek(seek_offset, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   # If processing failed, we need to pause
                   # For long running job this will overwrite the default never-ending pause and

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -52,14 +52,17 @@ module Karafka
                   # If still not revoked and was throttled, we need to apply filtering logic
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
+
+                    # :seek and :pause are fully handled by handle_post_filtering
+                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                    return unless coordinator.filter.action == :skip
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
                     seek(last_group_message.offset + 1, false, reset_offset: false)
-                    resume
-                  else
-                    resume
                   end
+
+                  resume
                 else
                   # If processing failed, we need to pause
                   # For long running job this will overwrite the default never-ending pause and

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -73,8 +73,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              monitor.instrument('consumer.revoke', caller: self)
-              monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument("consumer.revoke", caller: self)
+              monitor.instrument("consumer.revoked", caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/recurring_tasks/consumer.rb

@@ -47,7 +47,7 @@ module Karafka
             type = payload[:type]
 
             case type
-            when 'schedule'
+            when "schedule"
               # If we're replaying data, we need to record the most recent stored state, so we
               # can use this data to fully initialize the scheduler
               @executor.update_state(payload) if @executor.replaying?
@@ -62,7 +62,7 @@ module Karafka
                 partition,
                 message.offset - 1
               )
-            when 'command'
+            when "command"
               @executor.apply_command(payload)
 
               next if @executor.replaying?

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -29,8 +29,8 @@ module Karafka
         class Config < Karafka::Contracts::Base
           configure do |config|
             config.error_messages = YAML.safe_load_file(
-              File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-            ).fetch('en').fetch('validations').fetch('setup').fetch('config')
+              File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+            ).fetch("en").fetch("validations").fetch("setup").fetch("config")
           end
 
           nested(:recurring_tasks) do

data/lib/karafka/pro/recurring_tasks/contracts/task.rb

@@ -29,8 +29,8 @@ module Karafka
         class Task < Karafka::Contracts::Base
           configure do |config|
             config.error_messages = YAML.safe_load_file(
-              File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-            ).fetch('en').fetch('validations').fetch('recurring_tasks')
+              File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+            ).fetch("en").fetch("validations").fetch("recurring_tasks")
           end
 
           # Regexp to ensure all tasks ids are URL safe

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -34,7 +34,7 @@ module Karafka
           def schedule
             produce(
               topics.schedules.name,
-              'state:schedule',
+              "state:schedule",
               serializer.schedule(::Karafka::Pro::RecurringTasks.schedule)
             )
           end
@@ -87,7 +87,7 @@ module Karafka
               key: key,
               partition: 0,
               payload: payload,
-              headers: { 'zlib' => 'true' }
+              headers: { "zlib" => "true" }
             )
           end
         end

data/lib/karafka/pro/recurring_tasks/listener.rb

@@ -37,7 +37,7 @@ module Karafka
         #   are independent. It is not to replace the Web UI tracking but to just log failed
         #   executions in the same way as successful but just with the failure as an outcome.
         def on_error_occurred(event)
-          return unless event[:type] == 'recurring_tasks.task.execute.error'
+          return unless event[:type] == "recurring_tasks.task.execute.error"
 
           Dispatcher.log(event)
         end

data/lib/karafka/pro/recurring_tasks/matcher.rb

@@ -30,10 +30,10 @@ module Karafka
         # @return [Boolean] is this message dedicated to current process and is actionable
         def matches?(task, payload)
           # We only match commands
-          return false unless payload[:type] == 'command'
+          return false unless payload[:type] == "command"
 
           # * is a wildcard to match all for batch commands
-          return false unless payload[:task][:id] == '*' || payload[:task][:id] == task.id
+          return false unless payload[:task][:id] == "*" || payload[:task][:id] == task.id
 
           # Ignore messages that have different schema. This can happen in the middle of
           # upgrades of the framework. We ignore this not to risk compatibility issues

data/lib/karafka/pro/recurring_tasks/serializer.rb

@@ -26,7 +26,7 @@ module Karafka
       # Converts schedule command and log details into data we can dispatch to Kafka.
       class Serializer
         # Current recurring tasks related schema structure
-        SCHEMA_VERSION = '1.0'
+        SCHEMA_VERSION = "1.0"
 
         # Serializes and compresses the schedule with all its tasks and their execution state
         # @param schedule [Karafka::Pro::RecurringTasks::Schedule]
@@ -49,7 +49,7 @@ module Karafka
             schema_version: SCHEMA_VERSION,
             schedule_version: schedule.version,
             dispatched_at: Time.now.to_f,
-            type: 'schedule',
+            type: "schedule",
             tasks: tasks
           }
 
@@ -66,7 +66,7 @@ module Karafka
             schema_version: SCHEMA_VERSION,
             schedule_version: Karafka::Pro::RecurringTasks.schedule.version,
             dispatched_at: Time.now.to_f,
-            type: 'command',
+            type: "command",
             command: {
               name: command_name
             },
@@ -89,11 +89,11 @@ module Karafka
             schema_version: SCHEMA_VERSION,
             schedule_version: Karafka::Pro::RecurringTasks.schedule.version,
             dispatched_at: Time.now.to_f,
-            type: 'log',
+            type: "log",
             task: {
               id: task.id,
               time_taken: event.payload[:time] || -1,
-              result: event.payload.key?(:error) ? 'failure' : 'success'
+              result: event.payload.key?(:error) ? "failure" : "success"
             }
           }
 

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -31,7 +31,7 @@ module Karafka
 
           setting(:consumer_class, default: Consumer)
           setting(:deserializer, default: Deserializer.new)
-          setting(:group_id, default: 'karafka_recurring_tasks')
+          setting(:group_id, default: "karafka_recurring_tasks")
           # By default we will run the scheduling every 15 seconds since we provide a minute-based
           # precision
           setting(:interval, default: 15_000)
@@ -50,11 +50,11 @@ module Karafka
 
           setting(:topics) do
             setting(:schedules) do
-              setting(:name, default: 'karafka_recurring_tasks_schedules')
+              setting(:name, default: "karafka_recurring_tasks_schedules")
             end
 
             setting(:logs) do
-              setting(:name, default: 'karafka_recurring_tasks_logs')
+              setting(:name, default: "karafka_recurring_tasks_logs")
             end
           end
 

data/lib/karafka/pro/recurring_tasks/task.rb

@@ -103,7 +103,7 @@ module Karafka
         # Executes the given task and publishes appropriate notification bus events.
         def call
           monitor.instrument(
-            'recurring_tasks.task.executed',
+            "recurring_tasks.task.executed",
             task: self
           ) do
             # We check for presence of the `@executable` because user can define cron schedule
@@ -112,13 +112,13 @@ module Karafka
 
             execute
           end
-        rescue StandardError => e
+        rescue => e
           monitor.instrument(
-            'error.occurred',
+            "error.occurred",
             caller: self,
             error: e,
             task: self,
-            type: 'recurring_tasks.task.execute.error'
+            type: "recurring_tasks.task.execute.error"
           )
         ensure
           @trigger = false