phobos 1.8.0 → 1.8.1

data/lib/phobos/cli.rb

@@ -1,9 +1,10 @@
+# frozen_string_literal: true
+
 require 'thor'
 require 'phobos/cli/start'
 
 module Phobos
   module CLI
-
     def self.logger
       @logger ||= Logging.logger[self].tap do |l|
         l.appenders = [Logging.appenders.stdout]

data/lib/phobos/cli/runner.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 module Phobos
   module CLI
     class Runner
-
-      SIGNALS = %i( INT TERM QUIT ).freeze
+      SIGNALS = [:INT, :TERM, :QUIT].freeze
 
       def initialize
         @signal_queue = []
@@ -42,7 +43,6 @@ module Phobos
         writer.write_nonblock('.')
         signal_queue << signal
       end
-
     end
   end
 end

data/lib/phobos/cli/start.rb

@@ -1,17 +1,15 @@
+# frozen_string_literal: true
+
 require 'phobos/cli/runner'
 
 module Phobos
   module CLI
     class Start
       def initialize(options)
-        unless options[:skip_config]
-          @config_file = File.expand_path(options[:config])
-        end
+        @config_file = File.expand_path(options[:config]) unless options[:skip_config]
         @boot_file = File.expand_path(options[:boot])
 
-        if options[:listeners]
-          @listeners_file = File.expand_path(options[:listeners])
-        end
+        @listeners_file = File.expand_path(options[:listeners]) if options[:listeners]
       end
 
       def execute
@@ -22,9 +20,7 @@ module Phobos
           Phobos.configure(config_file)
         end
 
-        if listeners_file
-          Phobos.add_listeners(listeners_file)
-        end
+        Phobos.add_listeners(listeners_file) if listeners_file
 
         validate_listeners!
 
@@ -36,37 +32,33 @@ module Phobos
       attr_reader :config_file, :boot_file, :listeners_file
 
       def validate_config_file!
-        unless File.exist?(config_file)
-          Phobos::CLI.logger.error { Hash(message: "Config file not found (#{config_file})") }
-          exit(1)
-        end
+        File.exist?(config_file) || error_exit("Config file not found (#{config_file})")
       end
 
       def validate_listeners!
         Phobos.config.listeners.each do |listener|
-          handler_class = listener.handler
+          handler = listener.handler
 
-          begin
-            handler_class.constantize
-          rescue NameError
-            Phobos::CLI.logger.error { Hash(message: "Handler '#{handler_class}' not defined") }
-            exit(1)
-          end
+          Object.const_defined?(handler) || error_exit("Handler '#{handler}' not defined")
 
           delivery = listener.delivery
           if delivery.nil?
             Phobos::CLI.logger.warn do
-              Hash(message: "Delivery option should be specified, defaulting to 'batch' - specify this option to silence this message")
+              Hash(message: "Delivery option should be specified, defaulting to 'batch'"\
+                ' - specify this option to silence this message')
             end
           elsif !Listener::DELIVERY_OPTS.include?(delivery)
-            Phobos::CLI.logger.error do
-              Hash(message: "Invalid delivery option '#{delivery}'. Please specify one of: #{Listener::DELIVERY_OPTS.join(', ')}")
-            end
-            exit(1)
+            error_exit("Invalid delivery option '#{delivery}'. Please specify one of: "\
+              "#{Listener::DELIVERY_OPTS.join(', ')}")
           end
         end
       end
 
+      def error_exit(msg)
+        Phobos::CLI.logger.error { Hash(message: msg) }
+        exit(1)
+      end
+
       def load_boot_file
         load(boot_file) if File.exist?(boot_file)
       end

data/lib/phobos/constants.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Phobos
+  module Constants
+    LOG_DATE_PATTERN = '%Y-%m-%dT%H:%M:%S:%L%zZ'
+
+    KAFKA_CONSUMER_OPTS = [
+      :session_timeout,
+      :offset_commit_interval,
+      :offset_commit_threshold,
+      :heartbeat_interval,
+      :offset_retention_time
+    ].freeze
+
+    LISTENER_OPTS = [
+      :handler,
+      :group_id,
+      :topic,
+      :min_bytes,
+      :max_wait_time,
+      :force_encoding,
+      :start_from_beginning,
+      :max_bytes_per_partition,
+      :backoff,
+      :delivery,
+      :session_timeout,
+      :offset_commit_interval,
+      :offset_commit_threshold,
+      :heartbeat_interval,
+      :offset_retention_time
+    ].freeze
+  end
+end

data/lib/phobos/deep_struct.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Please use this with at least the same consideration as you would when using OpenStruct.
 # Right now we only use this to parse our internal configuration files. It is not meant to
 # be used on incoming data.
@@ -5,36 +7,33 @@ module Phobos
   class DeepStruct < OpenStruct
     # Based on
     # https://docs.omniref.com/ruby/2.3.0/files/lib/ostruct.rb#line=88
-    def initialize(hash=nil)
+    def initialize(hash = nil)
       @table = {}
       @hash_table = {}
 
-      if hash
-        hash.each_pair do |k, v|
-          k = k.to_sym
-          @table[k] = to_deep_struct(v)
-          @hash_table[k] = v
-        end
+      hash&.each_pair do |key, value|
+        key = key.to_sym
+        @table[key] = to_deep_struct(value)
+        @hash_table[key] = value
       end
     end
 
     def to_h
       @hash_table.dup
     end
-    alias_method :to_hash, :to_h
+    alias to_hash to_h
 
     private
 
-    def to_deep_struct(v)
-      case v
+    def to_deep_struct(value)
+      case value
       when Hash
-        self.class.new(v)
+        self.class.new(value)
       when Enumerable
-        v.map { |el| to_deep_struct(el) }
+        value.map { |el| to_deep_struct(el) }
       else
-        v
+        value
       end
     end
-    protected :to_deep_struct
   end
 end

data/lib/phobos/echo_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Phobos
   class EchoHandler
     include Phobos::Handler

data/lib/phobos/errors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Phobos
   class Error < StandardError; end
   class AbortError < Error; end

data/lib/phobos/executor.rb

@@ -1,23 +1,9 @@
+# frozen_string_literal: true
+
 module Phobos
   class Executor
     include Phobos::Instrumentation
-    LISTENER_OPTS = %i(
-      handler
-      group_id
-      topic
-      min_bytes
-      max_wait_time
-      force_encoding
-      start_from_beginning
-      max_bytes_per_partition
-      backoff
-      delivery
-      session_timeout
-      offset_commit_interval
-      offset_commit_threshold
-      heartbeat_interval
-      offset_retention_time
-    ).freeze
+    include Phobos::Log
 
     def initialize
       @threads = Concurrent::Array.new
@@ -26,7 +12,7 @@ module Phobos
         listener_configs = config.to_hash.deep_symbolize_keys
         max_concurrency = listener_configs[:max_concurrency] || 1
         Array.new(max_concurrency).map do
-          configs = listener_configs.select { |k| LISTENER_OPTS.include?(k) }
+          configs = listener_configs.select { |k| Constants::LISTENER_OPTS.include?(k) }
           Phobos::Listener.new(configs.merge(handler: handler_class))
         end
       end
@@ -51,10 +37,17 @@ module Phobos
 
     def stop
       return if @signal_to_stop
+
       instrument('executor.stop') do
         @signal_to_stop = true
         @listeners.each(&:stop)
-        @threads.select(&:alive?).each { |thread| thread.wakeup rescue nil }
+        @threads.select(&:alive?).each do |thread|
+          begin
+            thread.wakeup
+          rescue StandardError
+            nil
+          end
+        end
         @thread_pool&.shutdown
         @thread_pool&.wait_for_termination
         Phobos.logger.info { Hash(message: 'Executor stopped') }
@@ -63,44 +56,48 @@ module Phobos
 
     private
 
-    def error_metadata(e)
+    def error_metadata(exception)
       {
-        exception_class: e.class.name,
-        exception_message: e.message,
-        backtrace: e.backtrace
+        exception_class: exception.class.name,
+        exception_message: exception.message,
+        backtrace: exception.backtrace
       }
     end
 
+    # rubocop:disable Lint/RescueException
     def run_listener(listener)
       retry_count = 0
-      backoff = listener.create_exponential_backoff
 
       begin
         listener.start
       rescue Exception => e
-        #
-        # When "listener#start" is interrupted it's safe to assume that the consumer
-        # and the kafka client were properly stopped, it's safe to call start
-        # again
-        #
-        interval = backoff.interval_at(retry_count).round(2)
-        metadata = {
-          listener_id: listener.id,
-          retry_count: retry_count,
-          waiting_time: interval
-        }.merge(error_metadata(e))
-
-        instrument('executor.retry_listener_error', metadata) do
-          Phobos.logger.error { Hash(message: "Listener crashed, waiting #{interval}s (#{e.message})").merge(metadata)}
-          sleep interval
-        end
-
+        handle_crashed_listener(listener, e, retry_count)
         retry_count += 1
         retry unless @signal_to_stop
       end
     rescue Exception => e
-      Phobos.logger.error { Hash(message: "Failed to run listener (#{e.message})").merge(error_metadata(e)) }
+      log_error("Failed to run listener (#{e.message})", error_metadata(e))
       raise e
     end
+    # rubocop:enable Lint/RescueException
+
+    # When "listener#start" is interrupted it's safe to assume that the consumer
+    # and the kafka client were properly stopped, it's safe to call start
+    # again
+    def handle_crashed_listener(listener, error, retry_count)
+      backoff = listener.create_exponential_backoff
+      interval = backoff.interval_at(retry_count).round(2)
+
+      metadata = {
+        listener_id: listener.id,
+        retry_count: retry_count,
+        waiting_time: interval
+      }.merge(error_metadata(error))
+
+      instrument('executor.retry_listener_error', metadata) do
+        log_error("Listener crashed, waiting #{interval}s (#{error.message})", metadata)
+        sleep interval
+      end
+    end
   end
 end

data/lib/phobos/handler.rb

@@ -1,27 +1,27 @@
+# frozen_string_literal: true
+
 module Phobos
   module Handler
     def self.included(base)
       base.extend(ClassMethods)
     end
 
-    def before_consume(payload, metadata)
+    def before_consume(payload, _metadata)
       payload
     end
 
-    def consume(payload, metadata)
+    def consume(_payload, _metadata)
       raise NotImplementedError
     end
 
-    def around_consume(payload, metadata)
+    def around_consume(_payload, _metadata)
       yield
     end
 
     module ClassMethods
-      def start(kafka_client)
-      end
+      def start(kafka_client); end
 
-      def stop
-      end
+      def stop; end
     end
   end
 end

data/lib/phobos/instrumentation.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/notifications'
 
 module Phobos
@@ -15,8 +17,8 @@ module Phobos
     end
 
     def instrument(event, extra = {})
-      ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |extra|
-        yield(extra) if block_given?
+      ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |args|
+        yield(args) if block_given?
       end
     end
   end

data/lib/phobos/listener.rb

@@ -1,30 +1,24 @@
+# frozen_string_literal: true
+
 module Phobos
+  # rubocop:disable Metrics/ParameterLists, Metrics/ClassLength
   class Listener
     include Phobos::Instrumentation
+    include Phobos::Log
 
-    KAFKA_CONSUMER_OPTS = %i(
-      session_timeout
-      offset_commit_interval
-      offset_commit_threshold
-      heartbeat_interval
-      offset_retention_time
-    ).freeze
-
-    DEFAULT_MAX_BYTES_PER_PARTITION = 1048576 # 1 MB
+    DEFAULT_MAX_BYTES_PER_PARTITION = 1_048_576 # 1 MB
     DELIVERY_OPTS = %w[batch message].freeze
 
     attr_reader :group_id, :topic, :id
     attr_reader :handler_class, :encoding
 
-    def initialize(handler:, group_id:, topic:, min_bytes: nil,
-                   max_wait_time: nil, force_encoding: nil,
-                   start_from_beginning: true, backoff: nil,
-                   delivery: 'batch',
-                   max_bytes_per_partition: DEFAULT_MAX_BYTES_PER_PARTITION,
+    # rubocop:disable Metrics/MethodLength
+    def initialize(handler:, group_id:, topic:, min_bytes: nil, max_wait_time: nil,
+                   force_encoding: nil, start_from_beginning: true, backoff: nil,
+                   delivery: 'batch', max_bytes_per_partition: DEFAULT_MAX_BYTES_PER_PARTITION,
                    session_timeout: nil, offset_commit_interval: nil,
                    heartbeat_interval: nil, offset_commit_threshold: nil,
-                   offset_retention_time: nil
-                  )
+                   offset_retention_time: nil)
       @id = SecureRandom.hex[0...6]
       @handler_class = handler
       @group_id = group_id
@@ -32,14 +26,11 @@ module Phobos
       @backoff = backoff
       @delivery = delivery.to_s
       @subscribe_opts = {
-        start_from_beginning: start_from_beginning,
-        max_bytes_per_partition: max_bytes_per_partition
+        start_from_beginning: start_from_beginning, max_bytes_per_partition: max_bytes_per_partition
       }
       @kafka_consumer_opts = compact(
-        session_timeout: session_timeout,
-        offset_commit_interval: offset_commit_interval,
-        heartbeat_interval: heartbeat_interval,
-        offset_retention_time: offset_retention_time,
+        session_timeout: session_timeout, offset_retention_time: offset_retention_time,
+        offset_commit_interval: offset_commit_interval, heartbeat_interval: heartbeat_interval,
         offset_commit_threshold: offset_commit_threshold
       )
       @encoding = Encoding.const_get(force_encoding.to_sym) if force_encoding
@@ -47,9 +38,59 @@ module Phobos
       @kafka_client = Phobos.create_kafka_client
       @producer_enabled = @handler_class.ancestors.include?(Phobos::Producer)
     end
+    # rubocop:enable Metrics/MethodLength
 
     def start
       @signal_to_stop = false
+
+      start_listener
+
+      begin
+        start_consumer_loop
+      rescue Kafka::ProcessingError, Phobos::AbortError
+        # Abort is an exception to prevent the consumer from committing the offset.
+        # Since "listener" had a message being retried while "stop" was called
+        # it's wise to not commit the batch offset to avoid data loss. This will
+        # cause some messages to be reprocessed
+        instrument('listener.retry_aborted', listener_metadata) do
+          log_info('Retry loop aborted, listener is shutting down', listener_metadata)
+        end
+      end
+    ensure
+      stop_listener
+    end
+
+    def stop
+      return if should_stop?
+
+      instrument('listener.stopping', listener_metadata) do
+        log_info('Listener stopping', listener_metadata)
+        @consumer&.stop
+        @signal_to_stop = true
+      end
+    end
+
+    def create_exponential_backoff
+      Phobos.create_exponential_backoff(@backoff)
+    end
+
+    def should_stop?
+      @signal_to_stop == true
+    end
+
+    def send_heartbeat_if_necessary
+      raise Phobos::AbortError if should_stop?
+
+      @consumer&.send_heartbeat_if_necessary
+    end
+
+    private
+
+    def listener_metadata
+      { listener_id: id, group_id: group_id, topic: topic, handler: handler_class.to_s }
+    end
+
+    def start_listener
       instrument('listener.start', listener_metadata) do
         @consumer = create_kafka_consumer
         @consumer.subscribe(topic, @subscribe_opts)
@@ -58,25 +99,14 @@ module Phobos
         # since "start" blocks a thread might be used to call it
         @handler_class.producer.configure_kafka_client(@kafka_client) if @producer_enabled
 
-        instrument('listener.start_handler', listener_metadata) { @handler_class.start(@kafka_client) }
-        Phobos.logger.info { Hash(message: 'Listener started').merge(listener_metadata) }
-      end
-
-      begin
-        @delivery == 'batch' ? consume_each_batch : consume_each_message
-
-      # Abort is an exception to prevent the consumer from committing the offset.
-      # Since "listener" had a message being retried while "stop" was called
-      # it's wise to not commit the batch offset to avoid data loss. This will
-      # cause some messages to be reprocessed
-      #
-      rescue Kafka::ProcessingError, Phobos::AbortError
-        instrument('listener.retry_aborted', listener_metadata) do
-          Phobos.logger.info({ message: 'Retry loop aborted, listener is shutting down' }.merge(listener_metadata))
+        instrument('listener.start_handler', listener_metadata) do
+          @handler_class.start(@kafka_client)
         end
+        log_info('Listener started', listener_metadata)
       end
+    end
 
-    ensure
+    def stop_listener
       instrument('listener.stop', listener_metadata) do
         instrument('listener.stop_handler', listener_metadata) { @handler_class.stop }
 
@@ -88,12 +118,14 @@ module Phobos
         end
 
         @kafka_client.close
-        if should_stop?
-          Phobos.logger.info { Hash(message: 'Listener stopped').merge(listener_metadata) }
-        end
+        log_info('Listener stopped', listener_metadata) if should_stop?
       end
     end
 
+    def start_consumer_loop
+      @delivery == 'batch' ? consume_each_batch : consume_each_message
+    end
+
     def consume_each_batch
       @consumer.each_batch(@message_processing_opts) do |batch|
         batch_processor = Phobos::Actions::ProcessBatch.new(
@@ -103,8 +135,8 @@ module Phobos
         )
 
         batch_processor.execute
-        Phobos.logger.debug { Hash(message: 'Committed offset').merge(batch_processor.metadata) }
-        return if should_stop?
+        log_debug('Committed offset', batch_processor.metadata)
+        return nil if should_stop?
       end
     end
 
@@ -117,41 +149,15 @@ module Phobos
         )
 
         message_processor.execute
-        Phobos.logger.debug { Hash(message: 'Committed offset').merge(message_processor.metadata) }
-        return if should_stop?
-      end
-    end
-
-    def stop
-      return if should_stop?
-      instrument('listener.stopping', listener_metadata) do
-        Phobos.logger.info { Hash(message: 'Listener stopping').merge(listener_metadata) }
-        @consumer&.stop
-        @signal_to_stop = true
+        log_debug('Committed offset', message_processor.metadata)
+        return nil if should_stop?
       end
     end
 
-    def create_exponential_backoff
-      Phobos.create_exponential_backoff(@backoff)
-    end
-
-    def should_stop?
-      @signal_to_stop == true
-    end
-
-    def send_heartbeat_if_necessary
-      raise Phobos::AbortError if should_stop?
-      @consumer&.send_heartbeat_if_necessary
-    end
-
-    private
-
-    def listener_metadata
-      { listener_id: id, group_id: group_id, topic: topic, handler: handler_class.to_s }
-    end
-
     def create_kafka_consumer
-      configs = Phobos.config.consumer_hash.select { |k| KAFKA_CONSUMER_OPTS.include?(k) }
+      configs = Phobos.config.consumer_hash.select do |k|
+        Constants::KAFKA_CONSUMER_OPTS.include?(k)
+      end
       configs.merge!(@kafka_consumer_opts)
       @kafka_client.consumer({ group_id: group_id }.merge(configs))
     end
@@ -160,4 +166,5 @@ module Phobos
       hash.delete_if { |_, v| v.nil? }
     end
   end
+  # rubocop:enable Metrics/ParameterLists, Metrics/ClassLength
 end