phobos 1.8.1 → 1.8.2.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2521d4d12e282bf2f2fc1ae720c52cfdab4637e2464840e196c7fd275f316c9d
-  data.tar.gz: 067d38a797a93bd7300e31c34b16e2bdea1dfd43ff6eddaaeac678b33ce9944c
+  metadata.gz: 6610d5e383b53c4d2f926b4ab10ae33c260215fe2de0520132fa38e6b51089d8
+  data.tar.gz: '0791b9c0fc4d4b10f878607bddc1374c359bbe627386375ce48ea3ff6996329d'
 SHA512:
-  metadata.gz: a2808ac665d54a407da693e3ff0c8635ce0ef0daaf9815acf22c47628f268866cb350925a77477631f338cff360f79cc4e55b113a83c66e582333698bc19c800
-  data.tar.gz: 896136ecf3e4e41eb255c043b3982c55eb008a057ea97b47082b44826f2c59c8c525255f2fa3b9f2271df3e494dcc826d17c1136d7308c10b43bc63291fa4121
+  metadata.gz: dc15fe9776e8aff478e3263a9d8d16b9d693baff6259ae62bdbf1e229b7f8c8bfb345fdf7b9603583ca5405642e1424461c2523ef4bdbcd9b85bd80140a03e6d
+  data.tar.gz: 782ca0a9838cf38a6ae37c92b33d3ce2f4692438a837aeeefabbea629dd6d5b6c595a6cdc7932e804882d29854784583d356574f8adfaa25613fe093e42cf10c

data/.travis.yml

@@ -11,7 +11,7 @@ services:
 env:
   global:
     - DEFAULT_TIMEOUT=20
-    - CC_TEST_REPORTER_ID=ce08ec7297e0161c25b6b0c86e7bd4b9eb85b15de9e2a17fe419b8439aa40def
+    - CC_TEST_REPORTER_ID=a990fd40ff4421e8b28167232242a1d6c6638f9907fc0b80eb90f82f88fcfd81
 
 before_install:
   - env

data/CHANGELOG.md

@@ -4,7 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## UNRELEASED
+## [1.8.2-beta1] - 2019-03-13
+
+- Added `BatchHandler` to consume messages in batches on the business
+  layer.
 
 ## [1.8.1] - 2018-11-23
 ### Added

data/README.md

@@ -1,10 +1,8 @@
 ![Phobos](https://raw.githubusercontent.com/klarna/phobos/master/logo.png)
 
-[![Build Status](https://travis-ci.org/klarna/phobos.svg?branch=master)](https://travis-ci.org/klarna/phobos)
-[![Maintainability](https://api.codeclimate.com/v1/badges/2d00845fc6e7e83df6e7/maintainability)](https://codeclimate.com/github/klarna/phobos/maintainability)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/2d00845fc6e7e83df6e7/test_coverage)](https://codeclimate.com/github/klarna/phobos/test_coverage)
-[![Depfu](https://badges.depfu.com/badges/57da3d5ff1da449cf8739cfe30b8d2f8/count.svg)](https://depfu.com/github/klarna/phobos?project=Bundler)
-[![Chat with us on Discord](https://discordapp.com/api/guilds/379938130326847488/widget.png)](https://discord.gg/rfMUBVD)
+[![Build Status](https://travis-ci.com/phobos/phobos.svg?branch=master)](https://travis-ci.com/phobos/phobos)
+[![Maintainability](https://api.codeclimate.com/v1/badges/e3814d747c91247b24c6/maintainability)](https://codeclimate.com/github/phobos/phobos/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/e3814d747c91247b24c6/test_coverage)](https://codeclimate.com/github/phobos/phobos/test_coverage)
 
 # Phobos
 
@@ -214,6 +212,49 @@ The hander life cycle can be illustrated as:
 or optionally,
 
   `.start` -> `#before_consume` -> `#around_consume` [ `#consume` ] -> `.stop`
+  
+#### Batch Consumption
+
+In addition to the regular handler, Phobos provides a `BatchHandler`. The
+basic ideas are identical, except that instead of being passed a single message
+at a time, the `BatchHandler` is passed a batch of messages. All methods
+follow the same pattern as the regular handler except that they each
+end in `_batch` and are passed an array of `Phobos::BatchMessage`s instead
+of a single payload.
+
+To enable handling of batches on the consumer side, you must specify
+a delivery method of `inline_batch` in [phobos.yml](config/phobos.yml.example),
+and your handler must include `BatchHandler`. Using a delivery method of `batch`
+assumes that you are still processing the messages one at a time and should
+use `Handler`.
+
+```ruby
+class MyBatchHandler
+  includes Phobos::BatchHandler
+  
+  def before_consume_batch(payloads, metadata)
+    payloads.each do |p|
+      p.payload[:timestamp] = Time.zone.now
+    end
+  end
+  
+  def around_consume_batch(payloads, metadata)
+    yield
+  end
+  
+  def consume_batch(payloads, metadata)
+    payloads.each do |p|
+      logger.info("Got payload #{p.payload}, #{p.partition}, #{p.offset}, #{p.key}, #{p.payload[:timestamp]}")
+    end
+  end
+  
+end
+```
+
+Note that retry logic will happen on the *batch* level in this case. If you are
+processing messages individually and an error happens in the middle, Phobos's
+retry logic will retry the entire batch. If this is not the behavior you want,
+consider using `batch` instead of `inline_batch`.
 
 ### <a name="usage-producing-messages-to-kafka"></a> Producing messages to Kafka
 
@@ -436,7 +477,7 @@ end
     * partition
     * offset_lag
     * highwater_mark_offset
-  * `listener.process_message` is sent after process a message. It includes the following payload:
+  * `listener.process_message` is sent after processing a message. It includes the following payload:
     * listener_id
     * group_id
     * topic
@@ -445,7 +486,16 @@ end
     * partition
     * offset
     * retry_count
-  * `listener.retry_handler_error` is sent after waited for `handler#consume` retry. It includes the following payload:
+  * `listener.process_batch_inline` is sent after processing a batch with `batch_inline` mode. It includes the following payload:
+    * listener_id
+    * group_id
+    * topic
+    * handler
+    * batch_size
+    * partition
+    * offset_lag
+    * retry_count
+  * `listener.retry_handler_error` is sent after waiting for `handler#consume` retry. It includes the following payload:
     * listener_id
     * group_id
     * topic
@@ -458,22 +508,35 @@ end
     * exception_class
     * exception_message
     * backtrace
+  * `listener.retry_handler_error_batch` is sent after waiting for `handler#consume_batch` retry. It includes the following payload:
+    * listener_id
+    * group_id
+    * topic
+    * handler
+    * batch_size
+    * partition
+    * offset_lag
+    * retry_count
+    * waiting_time
+    * exception_class
+    * exception_message
+    * backtrace
   * `listener.retry_aborted` is sent after waiting for a retry but the listener was stopped before the retry happened. It includes the following payload:
     * listener_id
     * group_id
     * topic
     * handler
-  * `listener.stopping` is sent when the listener receives signal to stop
+  * `listener.stopping` is sent when the listener receives signal to stop.
     * listener_id
     * group_id
     * topic
     * handler
-  * `listener.stop_handler` is sent after stopping the handler
+  * `listener.stop_handler` is sent after stopping the handler.
     * listener_id
     * group_id
     * topic
     * handler
-  * `listener.stop` is send after stopping the listener
+  * `listener.stop` is send after stopping the listener.
     * listener_id
     * group_id
     * topic

data/config/phobos.yml.example

@@ -107,7 +107,11 @@ listeners:
     #  - `batch` will yield batches from Ruby Kafka using `each_batch`, and commit/heartbeat at every consumed batch.
     #     Due to implementation in Ruby Kafka, it should be noted that when configuring large batch sizes in combination with taking long time to consume messages,
     #     your consumers might get kicked from the Kafka cluster for not sending a heartbeat within the expected interval.
-    #     Take this into consideration when determining your configuration.
+    #     Take this into consideration when determining your configuration. You can send heartbeats manually while
+    #     processing your messages if necessary.
+    #   - `inline_batch` also uses `each_batch`, but will pass the entire batch to your handler instead
+    #     of one message at a time. To use this method, you should include Phobos::BatchHandler
+    #     instead of Phobos::Handler so that you can make use of the `consume_batch` etc. methods.
     # Note: Ultimately commit/heartbeart will depend on the offset commit options and the heartbeat interval.
     delivery: batch
     # Use this if custom backoff is required for a listener

data/lib/phobos.rb

@@ -22,8 +22,10 @@ require 'phobos/errors'
 require 'phobos/listener'
 require 'phobos/actions/process_batch'
 require 'phobos/actions/process_message'
+require 'phobos/actions/process_batch_inline'
 require 'phobos/producer'
 require 'phobos/handler'
+require 'phobos/batch_handler'
 require 'phobos/echo_handler'
 require 'phobos/executor'
 

data/lib/phobos/actions/process_batch_inline.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'phobos/batch_message'
+require 'phobos/processor'
+
+module Phobos
+  module Actions
+    class ProcessBatchInline
+      include Phobos::Processor
+
+      attr_reader :metadata
+
+      def initialize(listener:, batch:, metadata:)
+        @listener = listener
+        @batch = batch
+        @listener = listener
+        @batch = batch
+        @metadata = metadata.merge(
+          batch_size: batch.messages.count,
+          partition: batch.partition,
+          offset_lag: batch.offset_lag,
+          retry_count: 0
+        )
+      end
+
+      def execute
+        payloads = @batch.messages.map do |message|
+          Phobos::BatchMessage.new(
+            key: message.key,
+            partition: message.partition,
+            offset: message.offset,
+            payload: force_encoding(message.value)
+          )
+        end
+
+        begin
+          process_batch(payloads)
+        rescue StandardError => e
+          handle_error(e, 'listener.retry_handler_error_batch',
+                       "error processing inline batch, waiting #{backoff_interval}s")
+          retry
+        end
+      end
+
+      private
+
+      def process_batch(payloads)
+        instrument('listener.process_batch_inline', @metadata) do |_metadata|
+          handler = @listener.handler_class.new
+
+          preprocessed_payloads = handler.before_consume_batch(payloads, @metadata)
+          consume_block = proc { handler.consume_batch(preprocessed_payloads, @metadata) }
+
+          handler.around_consume_batch(preprocessed_payloads, @metadata, &consume_block)
+        end
+      end
+    end
+  end
+end

data/lib/phobos/actions/process_message.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
+require 'phobos/processor'
+
 module Phobos
   module Actions
     class ProcessMessage
-      MAX_SLEEP_INTERVAL = 3
-
-      include Phobos::Instrumentation
+      include Phobos::Processor
 
       attr_reader :metadata
 
@@ -26,29 +26,14 @@ module Phobos
         begin
           process_message(payload)
         rescue StandardError => e
-          handle_error(e)
+          handle_error(e, 'listener.retry_handler_error',
+                       "error processing message, waiting #{backoff_interval}s")
           retry
         end
       end
 
-      def snooze(interval)
-        remaining_interval = interval
-
-        @listener.send_heartbeat_if_necessary
-
-        while remaining_interval.positive?
-          sleep [remaining_interval, MAX_SLEEP_INTERVAL].min
-          remaining_interval -= MAX_SLEEP_INTERVAL
-          @listener.send_heartbeat_if_necessary
-        end
-      end
-
       private
 
-      def force_encoding(value)
-        @listener.encoding ? value&.force_encoding(@listener.encoding) : value
-      end
-
       def process_message(payload)
         instrument('listener.process_message', @metadata) do
           handler = @listener.handler_class.new
@@ -75,43 +60,6 @@ module Phobos
           'please update your consumer. This will not be backwards compatible in the future.')
         handler.before_consume(payload)
       end
-
-      def handle_error(error)
-        error_hash = {
-          waiting_time: backoff_interval,
-          exception_class: error.class.name,
-          exception_message: error.message,
-          backtrace: error.backtrace
-        }
-
-        instrument('listener.retry_handler_error', error_hash.merge(@metadata)) do
-          Phobos.logger.error do
-            { message: "error processing message, waiting #{backoff_interval}s" }
-              .merge(error_hash)
-              .merge(@metadata)
-          end
-
-          snooze(backoff_interval)
-        end
-
-        increment_retry_count
-      end
-
-      def retry_count
-        @metadata[:retry_count]
-      end
-
-      def increment_retry_count
-        @metadata[:retry_count] = retry_count + 1
-      end
-
-      def backoff
-        @backoff ||= @listener.create_exponential_backoff
-      end
-
-      def backoff_interval
-        backoff.interval_at(retry_count).round(2)
-      end
     end
   end
 end

data/lib/phobos/batch_handler.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Phobos
+  module BatchHandler
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def before_consume_batch(payloads, _metadata)
+      payloads
+    end
+
+    def consume_batch(_payloads, _metadata)
+      raise NotImplementedError
+    end
+
+    def around_consume_batch(_payloads, _metadata)
+      yield
+    end
+
+    module ClassMethods
+      def start(kafka_client); end
+
+      def stop; end
+    end
+  end
+end

data/lib/phobos/batch_message.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Phobos
+  class BatchMessage
+    attr_accessor :key, :partition, :offset, :payload
+
+    def initialize(key:, partition:, offset:, payload:)
+      @key = key
+      @partition = partition
+      @offset = offset
+      @payload = payload
+    end
+
+    def ==(other)
+      [:key, :partition, :offset, :payload].all? do |s|
+        public_send(s) == other.public_send(s)
+      end
+    end
+  end
+end

data/lib/phobos/cli/start.rb

@@ -35,11 +35,15 @@ module Phobos
         File.exist?(config_file) || error_exit("Config file not found (#{config_file})")
       end
 
-      def validate_listeners!
+      def validate_listeners! # rubocop:disable Metrics/MethodLength
         Phobos.config.listeners.each do |listener|
           handler = listener.handler
 
-          Object.const_defined?(handler) || error_exit("Handler '#{handler}' not defined")
+          begin
+            handler.constantize
+          rescue NameError
+            error_exit("Handler '#{handler}' not defined")
+          end
 
           delivery = listener.delivery
           if delivery.nil?

data/lib/phobos/listener.rb

@@ -7,7 +7,7 @@ module Phobos
     include Phobos::Log
 
     DEFAULT_MAX_BYTES_PER_PARTITION = 1_048_576 # 1 MB
-    DELIVERY_OPTS = %w[batch message].freeze
+    DELIVERY_OPTS = %w[batch message inline_batch].freeze
 
     attr_reader :group_id, :topic, :id
     attr_reader :handler_class, :encoding
@@ -123,7 +123,15 @@ module Phobos
     end
 
     def start_consumer_loop
-      @delivery == 'batch' ? consume_each_batch : consume_each_message
+      # validate batch handling
+      case @delivery
+      when 'batch'
+        consume_each_batch
+      when 'inline_batch'
+        consume_each_batch_inline
+      else
+        consume_each_message
+      end
     end
 
     def consume_each_batch
@@ -140,6 +148,20 @@ module Phobos
       end
     end
 
+    def consume_each_batch_inline
+      @consumer.each_batch(@message_processing_opts) do |batch|
+        batch_processor = Phobos::Actions::ProcessBatchInline.new(
+          listener: self,
+          batch: batch,
+          metadata: listener_metadata
+        )
+
+        batch_processor.execute
+        log_debug('Committed offset', batch_processor.metadata)
+        return nil if should_stop?
+      end
+    end
+
     def consume_each_message
       @consumer.each_message(@message_processing_opts) do |message|
         message_processor = Phobos::Actions::ProcessMessage.new(

data/lib/phobos/processor.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Phobos
+  module Processor
+    extend ActiveSupport::Concern
+    include Phobos::Instrumentation
+
+    MAX_SLEEP_INTERVAL = 3
+
+    def snooze(interval)
+      remaining_interval = interval
+
+      @listener.send_heartbeat_if_necessary
+
+      while remaining_interval.positive?
+        sleep [remaining_interval, MAX_SLEEP_INTERVAL].min
+        remaining_interval -= MAX_SLEEP_INTERVAL
+        @listener.send_heartbeat_if_necessary
+      end
+    end
+
+    private
+
+    def force_encoding(value)
+      @listener.encoding ? value&.force_encoding(@listener.encoding) : value
+    end
+
+    def handle_error(error, instrumentation_key, error_message)
+      error_hash = {
+        waiting_time: backoff_interval,
+        exception_class: error.class.name,
+        exception_message: error.message,
+        backtrace: error.backtrace
+      }
+
+      instrument(instrumentation_key, error_hash.merge(@metadata)) do
+        Phobos.logger.error do
+          { message: error_message }
+            .merge(error_hash)
+            .merge(@metadata)
+        end
+
+        snooze(backoff_interval)
+      end
+
+      increment_retry_count
+    end
+
+    def retry_count
+      @metadata[:retry_count]
+    end
+
+    def increment_retry_count
+      @metadata[:retry_count] = retry_count + 1
+    end
+
+    def backoff
+      @backoff ||= @listener.create_exponential_backoff
+    end
+
+    def backoff_interval
+      backoff.interval_at(retry_count).round(2)
+    end
+  end
+end

data/lib/phobos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phobos
-  VERSION = '1.8.1'
+  VERSION = '1.8.2-beta1'
 end

data/phobos.gemspec

@@ -14,7 +14,8 @@ Gem::Specification.new do |spec|
     'Thiago R. Colucci',
     'Martin Svalin',
     'Francisco Juan',
-    'Tommy Gustafsson'
+    'Tommy Gustafsson',
+    'Daniel Orner'
   ]
   spec.email = [
     'ornelas.tulio@gmail.com',
@@ -23,7 +24,8 @@ Gem::Specification.new do |spec|
     'ticolucci@gmail.com',
     'martin@lite.nu',
     'francisco.juan@gmail.com',
-    'tommydgustafsson@gmail.com'
+    'tommydgustafsson@gmail.com',
+    'dmorner@gmail.com'
   ]
 
   spec.summary       = 'Simplifying Kafka for ruby apps'
@@ -53,7 +55,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'rake', '~> 12.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '0.60.0'
+  spec.add_development_dependency 'rubocop', '0.62.0'
   spec.add_development_dependency 'rubocop_rules'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'timecop'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 1.8.2.pre.beta1
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -11,10 +11,11 @@ authors:
 - Martin Svalin
 - Francisco Juan
 - Tommy Gustafsson
+- Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-23 00:00:00.000000000 Z
+date: 2019-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -78,14 +79,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.60.0
+        version: 0.62.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.60.0
+        version: 0.62.0
 - !ruby/object:Gem::Dependency
   name: rubocop_rules
   requirement: !ruby/object:Gem::Requirement
@@ -237,6 +238,7 @@ email:
 - martin@lite.nu
 - francisco.juan@gmail.com
 - tommydgustafsson@gmail.com
+- dmorner@gmail.com
 executables:
 - phobos
 extensions: []
@@ -268,7 +270,10 @@ files:
 - examples/publishing_messages_without_consumer.rb
 - lib/phobos.rb
 - lib/phobos/actions/process_batch.rb
+- lib/phobos/actions/process_batch_inline.rb
 - lib/phobos/actions/process_message.rb
+- lib/phobos/batch_handler.rb
+- lib/phobos/batch_message.rb
 - lib/phobos/cli.rb
 - lib/phobos/cli/runner.rb
 - lib/phobos/cli/start.rb
@@ -281,6 +286,7 @@ files:
 - lib/phobos/instrumentation.rb
 - lib/phobos/listener.rb
 - lib/phobos/log.rb
+- lib/phobos/processor.rb
 - lib/phobos/producer.rb
 - lib/phobos/test.rb
 - lib/phobos/test/helper.rb
@@ -304,12 +310,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Simplifying Kafka for ruby apps