phobos 1.8.0 → 1.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b37b13898c547e212449d8b598984b80ceb9796d377991b7dd73f6e6acfdeb8
-  data.tar.gz: 66d12f8461a5e3e796b47364644a948c2f1c030d227f61a6dd675baf7e368f62
+  metadata.gz: 2521d4d12e282bf2f2fc1ae720c52cfdab4637e2464840e196c7fd275f316c9d
+  data.tar.gz: 067d38a797a93bd7300e31c34b16e2bdea1dfd43ff6eddaaeac678b33ce9944c
 SHA512:
-  metadata.gz: 75e2ddf1d53ea28086f0442779c2b19f7b8e9d78d62c7b7f4257fdf59d37f70d145fc014da37678d3023a4395c7b84b1c6d81145187482344af493114673931d
-  data.tar.gz: da67af5cee56a7867f99f0353b847fd8d1faf23dd7f24e539fc6a90514b9e5c3c38bef04f29d03cbd4344242818127d1f69aafb4bd36d1ac11321179b75d7cb4
+  metadata.gz: a2808ac665d54a407da693e3ff0c8635ce0ef0daaf9815acf22c47628f268866cb350925a77477631f338cff360f79cc4e55b113a83c66e582333698bc19c800
+  data.tar.gz: 896136ecf3e4e41eb255c043b3982c55eb008a057ea97b47082b44826f2c59c8c525255f2fa3b9f2271df3e494dcc826d17c1136d7308c10b43bc63291fa4121

data/.rubocop.yml

@@ -0,0 +1,26 @@
+###########################
+# Configuration for rubocop
+# in .rubocop.yml
+
+##############
+# Global rules
+# see .rubocop_common.yml
+
+##############
+# Inherit default rules first, and then override those rules with
+# our violation whitelist.
+inherit_from:
+  - .rubocop_common.yml
+  - .rubocop_todo.yml
+
+##############
+# Project specific overrides here, example:
+# Metrics/BlockLength:
+#   Exclude:
+#    - 'tasks/the_huge_task.rake'
+
+AllCops:
+  Exclude:
+    - spec/**/*
+    - examples/**/*
+  TargetRubyVersion: 2.3

data/.rubocop_common.yml

@@ -0,0 +1,29 @@
+##############
+# Global rules
+AllCops:
+  Exclude:
+    - db/**/*
+  TargetRubyVersion: 2.3
+
+Rails:
+  Enabled: false
+
+Style/SymbolArray:
+  EnforcedStyle: brackets
+
+Metrics/LineLength:
+  Max: 100
+
+Metrics/BlockLength:
+  Exclude:
+   - '*.gemspec'
+   - 'spec/**/*.rb'
+
+Documentation:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 15
+
+Metrics/AbcSize:
+  Max: 17

data/.rubocop_todo.yml

@@ -0,0 +1,7 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2018-10-22 14:47:09 +0200 using RuboCop version 0.59.2.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.

data/.rubosync.yml

@@ -0,0 +1,2 @@
+git:
+  repo: git@github.com:phobos/shared.git

data/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## UNRELEASED
 
+## [1.8.1] - 2018-11-23
+### Added
+- Added ability to send partition keys separate from messsage keys.
+
 ## [1.8.0] - 2018-07-22
 ### Added
 - Possibility to configure a custom logger #81

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in phobos.gemspec

data/README.md

@@ -338,21 +338,21 @@ The configuration file is organized in 6 sections. Take a look at the example fi
 
 The file will be parsed through ERB so ERB syntax/file extension is supported beside the YML format.
 
-__logger__ configures the logger for all Phobos components. It automatically 
+__logger__ configures the logger for all Phobos components. It automatically
 outputs to `STDOUT` and it saves the log in the configured file.
 
-__kafka__ provides configurations for every `Kafka::Client` created over the application. 
+__kafka__ provides configurations for every `Kafka::Client` created over the application.
 All [options supported by  `ruby-kafka`][ruby-kafka-client] can be provided.
 
-__producer__ provides configurations for all producers created over the application, 
-the options are the same for regular and async producers. 
+__producer__ provides configurations for all producers created over the application,
+the options are the same for regular and async producers.
 All [options supported by  `ruby-kafka`][ruby-kafka-producer] can be provided.
 
-__consumer__ provides configurations for all consumer groups created over the application. 
+__consumer__ provides configurations for all consumer groups created over the application.
 All [options supported by  `ruby-kafka`][ruby-kafka-consumer] can be provided.
 
-__backoff__ Phobos provides automatic retries for your handlers. If an exception 
-is raised, the listener will retry following the back off configured here. 
+__backoff__ Phobos provides automatic retries for your handlers. If an exception
+is raised, the listener will retry following the back off configured here.
 Backoff can also be configured per listener.
 
 __listeners__ is the list of listeners configured. Each listener represents a consumer group.
@@ -537,6 +537,10 @@ end
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/klarna/phobos.
 
+## Linting
+
+Phobos projects Rubocop to lint the code, and in addition all projects use [Rubocop Rules](https://github.com/klippx/rubocop_rules) to maintain a shared rubocop configuration. Updates to the shared configurations are done in [phobos/shared](https://github.com/phobos/shared) repo, where you can also find instructions on how to apply the new settings to the Phobos projects.
+
 ## Acknowledgements
 
 Thanks to Sebastian Norde for the awesome logo!

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'irb'
 require 'bundler/setup'
@@ -11,7 +12,8 @@ require 'phobos'
 # require 'pry'
 # Pry.start
 
-config_path = ENV['CONFIG_PATH'] || (File.exist?('config/phobos.yml') ? 'config/phobos.yml' : 'config/phobos.yml.example')
+config_path = ENV['CONFIG_PATH'] ||
+              (File.exist?('config/phobos.yml') ? 'config/phobos.yml' : 'config/phobos.yml.example')
 Phobos.configure(config_path)
 
 IRB.start

data/bin/phobos

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
 

data/examples/handler_saving_events_database.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # This example assumes that you want to save all events in your database for
 # recovery purposes. The consumer will process the message and perform other
@@ -10,7 +12,7 @@ class HandlerSavingEventsDatabase
   include Phobos::Handler
   include Phobos::Producer
 
-  def self.around_consume(payload, metadata)
+  def self.around_consume(payload, _metadata)
     #
     # Let's assume `::from_message` will initialize our object with `payload`
     #
@@ -39,7 +41,7 @@ class HandlerSavingEventsDatabase
     end
   end
 
-  def consume(payload, metadata)
+  def consume(payload, _metadata)
     #
     # Process the event, it might index it to elasticsearch or notify other
     # system, you should process your message inside this method.

data/examples/handler_using_async_producer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # This example assumes you want to process the event and publish another
 # one to kafka. A new event is always published thus we want to use the async producer
@@ -9,7 +11,7 @@ class HandlerUsingAsyncProducer
 
   PUBLISH_TO 'another-topic'
 
-  def consume(payload, metadata)
+  def consume(payload, _metadata)
     producer.async_publish(PUBLISH_TO, "#{payload}-#{rand}")
   end
 end

data/examples/publishing_messages_without_consumer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # This example assumes you want to create a threaded kafka generator which
 # publish a stream of kafka messages without consuming them. It also shows
@@ -19,9 +21,9 @@ end
 # Trapping signals to properly stop this generator
 #
 @stop = false
-%i( INT TERM QUIT ).each do |signal|
+[:INT, :TERM, :QUIT].each do |signal|
   Signal.trap(signal) do
-    puts "Stopping"
+    puts 'Stopping'
     @stop = true
   end
 end
@@ -32,6 +34,7 @@ Thread.new do
 
     loop do
       break if @stop
+
       key = SecureRandom.uuid
       payload = Time.now.utc.to_json
 
@@ -50,7 +53,7 @@ Thread.new do
       # the producer can write to Kafka. Eventually we'll get some buffer overflows
       #
       rescue Kafka::BufferOverflow => e
-        puts "| waiting"
+        puts '| waiting'
         sleep(1)
         retry
       end
@@ -67,7 +70,8 @@ Thread.new do
       .async_producer_shutdown
 
     #
-    # Since no client was configured (we can do this with `MyProducer.producer.configure_kafka_client`)
+    # Since no client was configured (we can do this with
+    #   `MyProducer.producer.configure_kafka_client`)
     # we must get the auto generated one and close it properly
     #
     MyProducer

data/lib/phobos.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ostruct'
 require 'securerandom'
 require 'yaml'
@@ -13,6 +15,8 @@ require 'erb'
 
 require 'phobos/deep_struct'
 require 'phobos/version'
+require 'phobos/constants'
+require 'phobos/log'
 require 'phobos/instrumentation'
 require 'phobos/errors'
 require 'phobos/listener'
@@ -31,16 +35,15 @@ module Phobos
     attr_accessor :silence_log
 
     def configure(configuration)
-      @config = DeepStruct.new(fetch_settings(configuration))
+      @config = fetch_configuration(configuration)
       @config.class.send(:define_method, :producer_hash) { Phobos.config.producer&.to_hash }
       @config.class.send(:define_method, :consumer_hash) { Phobos.config.consumer&.to_hash }
       @config.listeners ||= []
       configure_logger
-      logger.info { Hash(message: 'Phobos configured', env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'N/A') }
     end
 
-    def add_listeners(listeners_configuration)
-      listeners_config = DeepStruct.new(fetch_settings(listeners_configuration))
+    def add_listeners(configuration)
+      listeners_config = fetch_configuration(configuration)
       @config.listeners += listeners_config.listeners
     end
 
@@ -55,61 +58,88 @@ module Phobos
       ExponentialBackoff.new(min, max).tap { |backoff| backoff.randomize_factor = rand }
     end
 
+    def deprecate(message)
+      warn "DEPRECATION WARNING: #{message} #{Kernel.caller.first}"
+    end
+
     # :nodoc:
     def configure_logger
-      ruby_kafka = config.logger.ruby_kafka
-
       Logging.backtrace(true)
       Logging.logger.root.level = silence_log ? :fatal : config.logger.level
-      appenders = logger_appenders
 
-      @ruby_kafka_logger = nil
+      configure_ruby_kafka_logger
+      configure_phobos_logger
 
-      if config.custom_kafka_logger
-        @ruby_kafka_logger = config.custom_kafka_logger
-      elsif ruby_kafka
-        @ruby_kafka_logger = Logging.logger['RubyKafka']
-        @ruby_kafka_logger.appenders = appenders
-        @ruby_kafka_logger.level = silence_log ? :fatal : ruby_kafka.level
+      logger.info do
+        Hash(message: 'Phobos configured', env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'N/A')
       end
+    end
+
+    private
+
+    def fetch_configuration(configuration)
+      DeepStruct.new(read_configuration(configuration))
+    end
+
+    def read_configuration(configuration)
+      return configuration.to_h if configuration.respond_to?(:to_h)
+
+      YAML.safe_load(
+        ERB.new(
+          File.read(File.expand_path(configuration))
+        ).result,
+        [Symbol],
+        [],
+        true
+      )
+    end
 
+    def configure_phobos_logger
       if config.custom_logger
         @logger = config.custom_logger
       else
         @logger = Logging.logger[self]
-        @logger.appenders = appenders
+        @logger.appenders = logger_appenders
       end
     end
 
-    def logger_appenders
-      date_pattern = '%Y-%m-%dT%H:%M:%S:%L%zZ'
-      json_layout = Logging.layouts.json(date_pattern: date_pattern)
-      log_file = config.logger.file
-      stdout_layout = if config.logger.stdout_json == true
-                        json_layout
-                      else
-                        Logging.layouts.pattern(date_pattern: date_pattern)
-                      end
+    def configure_ruby_kafka_logger
+      if config.custom_kafka_logger
+        @ruby_kafka_logger = config.custom_kafka_logger
+      elsif config.logger.ruby_kafka
+        @ruby_kafka_logger = Logging.logger['RubyKafka']
+        @ruby_kafka_logger.appenders = logger_appenders
+        @ruby_kafka_logger.level = silence_log ? :fatal : config.logger.ruby_kafka.level
+      else
+        @ruby_kafka_logger = nil
+      end
+    end
 
+    def logger_appenders
       appenders = [Logging.appenders.stdout(layout: stdout_layout)]
 
       if log_file
         FileUtils.mkdir_p(File.dirname(log_file))
         appenders << Logging.appenders.file(log_file, layout: json_layout)
       end
+
       appenders
     end
 
-    def deprecate(message)
-      warn "DEPRECATION WARNING: #{message} #{Kernel.caller.first}"
+    def log_file
+      config.logger.file
     end
 
-    private
-
-    def fetch_settings(configuration)
-      return configuration.to_h if configuration.respond_to?(:to_h)
+    def json_layout
+      Logging.layouts.json(date_pattern: Constants::LOG_DATE_PATTERN)
+    end
 
-      YAML.load(ERB.new(File.read(File.expand_path(configuration))).result)
+    def stdout_layout
+      if config.logger.stdout_json == true
+        json_layout
+      else
+        Logging.layouts.pattern(date_pattern: Constants::LOG_DATE_PATTERN)
+      end
     end
   end
 end

data/lib/phobos/actions/process_batch.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Phobos
   module Actions
     class ProcessBatch
@@ -17,7 +19,7 @@ module Phobos
       end
 
       def execute
-        instrument('listener.process_batch', @metadata) do |metadata|
+        instrument('listener.process_batch', @metadata) do |_metadata|
           @batch.messages.each do |message|
             Phobos::Actions::ProcessMessage.new(
               listener: @listener,

data/lib/phobos/actions/process_message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Phobos
   module Actions
     class ProcessMessage
@@ -19,31 +21,12 @@ module Phobos
       end
 
       def execute
-        backoff = @listener.create_exponential_backoff
         payload = force_encoding(@message.value)
 
         begin
           process_message(payload)
-        rescue => e
-          retry_count = @metadata[:retry_count]
-          interval = backoff.interval_at(retry_count).round(2)
-
-          error = {
-            waiting_time: interval,
-            exception_class: e.class.name,
-            exception_message: e.message,
-            backtrace: e.backtrace
-          }
-
-          instrument('listener.retry_handler_error', error.merge(@metadata)) do
-            Phobos.logger.error do
-              { message: "error processing message, waiting #{interval}s" }.merge(error).merge(@metadata)
-            end
-
-            snooze(interval)
-          end
-
-          @metadata.merge!(retry_count: retry_count + 1)
+        rescue StandardError => e
+          handle_error(e)
           retry
         end
       end
@@ -69,19 +52,14 @@ module Phobos
       def process_message(payload)
         instrument('listener.process_message', @metadata) do
           handler = @listener.handler_class.new
-          preprocessed_payload = begin
-            handler.before_consume(payload, @metadata)
-          rescue ArgumentError => e
-            Phobos.deprecate("before_consume now expects metadata as second argument, please update your consumer."\
-              " This will not be backwards compatible in the future.")
-            handler.before_consume(payload)
-          end
-          consume_block = Proc.new { handler.consume(preprocessed_payload, @metadata) }
+
+          preprocessed_payload = before_consume(handler, payload)
+          consume_block = proc { handler.consume(preprocessed_payload, @metadata) }
 
           if @listener.handler_class.respond_to?(:around_consume)
             # around_consume class method implementation
-            Phobos.deprecate("around_consume has been moved to instance method, please update your consumer."\
-              " This will not be backwards compatible in the future.")
+            Phobos.deprecate('around_consume has been moved to instance method, '\
+              'please update your consumer. This will not be backwards compatible in the future.')
             @listener.handler_class.around_consume(preprocessed_payload, @metadata, &consume_block)
           else
             # around_consume instance method implementation
@@ -89,6 +67,51 @@ module Phobos
           end
         end
       end
+
+      def before_consume(handler, payload)
+        handler.before_consume(payload, @metadata)
+      rescue ArgumentError
+        Phobos.deprecate('before_consume now expects metadata as second argument, '\
+          'please update your consumer. This will not be backwards compatible in the future.')
+        handler.before_consume(payload)
+      end
+
+      def handle_error(error)
+        error_hash = {
+          waiting_time: backoff_interval,
+          exception_class: error.class.name,
+          exception_message: error.message,
+          backtrace: error.backtrace
+        }
+
+        instrument('listener.retry_handler_error', error_hash.merge(@metadata)) do
+          Phobos.logger.error do
+            { message: "error processing message, waiting #{backoff_interval}s" }
+              .merge(error_hash)
+              .merge(@metadata)
+          end
+
+          snooze(backoff_interval)
+        end
+
+        increment_retry_count
+      end
+
+      def retry_count
+        @metadata[:retry_count]
+      end
+
+      def increment_retry_count
+        @metadata[:retry_count] = retry_count + 1
+      end
+
+      def backoff
+        @backoff ||= @listener.create_exponential_backoff
+      end
+
+      def backoff_interval
+        backoff.interval_at(retry_count).round(2)
+      end
     end
   end
 end