phobos 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 576b1c85a5d3f389caac3e09a2c1738bb2743eda
-  data.tar.gz: 5fc77dbf88b6bfc824245cb58c9a34558562d379
+  metadata.gz: 05b2968d8c1fc02f229055c4163bd4c4472a9c1c
+  data.tar.gz: cca82af11ae5d8aa4996a8802bc516eda2fd8b4a
 SHA512:
-  metadata.gz: 74ff72c5bd479ee5e495d8da7ccc97c2bbe998c5ef7d601ce245a1b5e0a0054c5327c1cb100798dfb0832956fe91c8611f2aa66735d3af90e06b8dc8a915d713
-  data.tar.gz: 9dbf97945d5bb802fec64af350af7749273e30b9fd000e1ce435b1bfb2780b5f2dfc99d03c3a4cb3eaa1fa8e8a53b3e16ea75431c5057381f7d0993b22cb04b6
+  metadata.gz: 61cb3f4940c594967b2c0c20a8ef3e7ac22c811baaea4dd8b2aa482c76aa7c4ae2d93d4a1986b91fc4a6b4b7d6cdae66cc63e23278646b20dbed5346cfa3f31d
+  data.tar.gz: a5fbf1313da2817be54c5213a64d0e3e9220d621fc1efd9230cc9285c926d8041a72badf4133679065a888891c8f54da04c80e94d62e9f792542c3c76d02ebb6

data/.dockerignore

@@ -0,0 +1,13 @@
+.idea
+.DS_Store
+.dockerignore
+.gitignore
+.travis.yml
+Dockerfile
+docker-compose.yml
+Gemfile.lock
+logo.png
+log/
+coverage/
+spec/examples.txt
+pkg/

data/.env

@@ -0,0 +1 @@
+DEFAULT_TIMEOUT=10

data/.travis.yml

@@ -0,0 +1,35 @@
+sudo: required
+language: ruby
+rvm:
+  - 2.4.1
+
+services:
+  - docker
+
+env:
+  global:
+    - DEFAULT_TIMEOUT=20
+    - CC_TEST_REPORTER_ID=ce08ec7297e0161c25b6b0c86e7bd4b9eb85b15de9e2a17fe419b8439aa40def
+
+before_install:
+  - env
+  - docker-compose --version
+  - docker --version
+  - docker-compose config
+  - docker-compose up -d --force-recreate kafka zookeeper
+  - docker-compose build test
+
+before_script:
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then ./cc-test-reporter before-build || echo "Skipping CC coverage before-build"; fi
+  - mkdir coverage/
+  - touch ./coverage/.resultset.json
+
+script:
+  - docker-compose run --rm test
+
+after_script:
+  - cat ./coverage/.resultset.json | sed "s|/opt/phobos|$PWD|" > ./coverage/.newresultset.json
+  - cp ./coverage/.newresultset.json ./coverage/.resultset.json
+  - if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT || echo "Skipping CC coverage after-build"; fi

data/CHANGELOG.md

@@ -4,45 +4,69 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## 1.5.0 (2017-10-25)
-
-- [enhancement] Add `before_consume` callback to support single point of decoding a message.
-- [enhancement] Add module `Phobos::Test::Helper` for testing, to test consumers with minimal setup required
-
-## 1.4.2 (2017-09-29)
-
-- [bugfix] Async publishing always delivers messages #33
-
-## 1.4.1 (2017-08-22)
-
-- [enhancement] Update dev dependencies to fix warnings for the new unified Integer class
-- [bugfix] Include the error `Kafka::ProcessingError` into the abort block
-
-## 1.4.0 (2017-08-21)
-
-- [enhancement] Add support for hash provided settings #30
-
-## 1.3.0 (2017-06-15)
-
-- [enhancement] Add support for erb syntax in phobos config file #26
-
-## 1.2.1 (2016-10-12)
-
-- [bugfix] Ensure JSON layout for log files
-
-## 1.2.0 (2016-10-10)
-
-- [enhancement] Log file can be disabled #20
-- [enhancement] Add a new extra (time_elapsed) for notifications "listener.process_message" and "listener.process_batch" #24
-- [enhancement] Add option to configure ruby-kafka logger #23
-
-## 1.1.0 (2016-09-02)
-
-- [enhancement] Removed Hashie as a dependency #12
-- [feature] Allow configuring consumers min_bytes & max_wait_time #15
-- [feature] Allow configuring producers max_queue_size, delivery_threshold & delivery_interval #16
-- [feature] Allow configuring force_encoding for message payload #18
-
-## 1.0.0 (2016-08-08)
-
-- Published on Github with full fledged consumers and producers
+## UNRELEASED
+
+## [1.6.0] - 2017-11-16
+### Added
+- Support for outputting logs as json #50
+- Make configuration, especially of listeners, more flexible. #31
+- Phobos Discord chat
+- Support for consuming `each_message` instead of `each_batch` via the delivery listener option. #21
+- Instantiate a single handler class instance and use that both for `consume` and `before_consume`. #47
+
+### Changed
+- Pin ruby-kafka version to < 0.5.0 #48
+- Update changelog to follow the [Keep a Changelog](http://keepachangelog.com/) structure
+
+## [1.5.0] - 2017-10-25
+### Added
+- Add `before_consume` callback to support single point of decoding a message klarna/phobos_db_checkpoint#34
+- Add `Phobos::Test::Helper` for testing, to test consumers with minimal setup required
+
+### Changed
+- Allow configuration of backoff per listener #35
+- Move container orchestration into docker-compose
+- Update docker images #38
+
+### Fixed
+- Make specs run locally #36
+
+## [1.4.2] - 2017-09-29
+### Fixed
+- Async publishing always delivers messages #33
+
+## [1.4.1] - 2017-08-22
+### Added
+- Update dev dependencies to fix warnings for the new unified Integer class
+
+### Fixed
+- Include the error `Kafka::ProcessingError` into the abort block
+
+## [1.4.0] - 2017-08-21
+### Added
+- Support for hash provided settings #30
+
+## [1.3.0] - 2017-06-15
+### Added
+- Support for ERB syntax in Phobos config file #26
+
+## [1.2.1] - 2016-10-12
+### Fixed
+- Ensure JSON layout for log files
+
+## [1.2.0] - 2016-10-10
+### Added
+- Log file can be disabled #20
+- Property (time_elapsed) available for notifications `listener.process_message` and `listener.process_batch` #24
+- Option to configure ruby-kafka logger #23
+
+## [1.1.0] - 2016-09-02
+### Added
+- Removed Hashie as a dependency #12
+- Allow configuring consumers min_bytes & max_wait_time #15
+- Allow configuring producers max_queue_size, delivery_threshold & delivery_interval #16
+- Allow configuring force_encoding for message payload #18
+
+## [1.0.0] - 2016-08-08
+### Added
+- Published on Github!

data/Dockerfile

@@ -1,13 +1,14 @@
-FROM ruby:2.3.1-alpine
+FROM ruby:2.4.1-alpine
 
 RUN apk update && apk upgrade && \
     apk add --no-cache bash git openssh build-base
+RUN gem install bundler -v 1.16.0
 
 WORKDIR /opt/phobos
 
 ADD Gemfile Gemfile
 ADD phobos.gemspec phobos.gemspec
 ADD lib/phobos/version.rb lib/phobos/version.rb
+RUN bundle install
 
-RUN ["bundle", "install"]
 ADD . .

data/README.md

@@ -1,7 +1,9 @@
 ![Phobos](https://raw.githubusercontent.com/klarna/phobos/master/logo.png)
 
-[![Circle CI](https://circleci.com/gh/klarna/phobos.svg?style=shield&circle-token=2289e0fe5bd934074597b32e7f8f0bc98ea0e3c7)](https://circleci.com/gh/klarna/phobos/tree/master)
-[![Coverage Status](https://coveralls.io/repos/github/klarna/phobos/badge.svg?branch=master)](https://coveralls.io/github/klarna/phobos?branch=master)
+[![Build Status](https://travis-ci.org/klarna/phobos.svg?branch=master)](https://travis-ci.org/klarna/phobos)
+[![Maintainability](https://api.codeclimate.com/v1/badges/2d00845fc6e7e83df6e7/maintainability)](https://codeclimate.com/github/klarna/phobos/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/2d00845fc6e7e83df6e7/test_coverage)](https://codeclimate.com/github/klarna/phobos/test_coverage)
+[![Chat with us on Discord](https://discordapp.com/api/guilds/379938130326847488/widget.png)](https://discord.gg/rfMUBVD)
 
 # Phobos
 
@@ -117,6 +119,14 @@ By default, the __start__ command will look for the configuration file at `confi
 ```sh
 $ phobos start -c /var/configs/my.yml -b /opt/apps/boot.rb
 ```
+
+You may also choose to configure phobos with a hash from within your boot file.
+In this case, disable loading the config file with the `--skip-config` option:
+
+```sh
+$ phobos start -b /opt/apps/boot.rb --skip-config
+```
+
 ### <a name="usage-consuming-messages-from-kafka"></a> Consuming messages from Kafka
 
 Messages from Kafka are consumed using __handlers__. You can use Phobos __executors__ or include it in your own project [as a library](#usage-as-library), but __handlers__ will always be used. To create a handler class, simply include the module `Phobos::Handler`. This module allows Phobos to manage the life cycle of your handler.
@@ -325,6 +335,20 @@ __listeners__ is the list of listeners configured, each listener represents a co
 [ruby-kafka-consumer]: http://www.rubydoc.info/gems/ruby-kafka/Kafka%2FClient%3Aconsumer
 [ruby-kafka-producer]: http://www.rubydoc.info/gems/ruby-kafka/Kafka%2FClient%3Aproducer
 
+#### Additional listener configuration
+
+In some cases it's useful to  share _most_ of the configuration between
+multiple phobos processes, but have each process run different listeners. In
+that case, a separate yaml file can be created and loaded with the `-l` flag.
+Example:
+
+```sh
+$ phobos start -c /var/configs/my.yml -l /var/configs/additional_listeners.yml
+```
+
+Note that the config file _must_ still specify a listeners section, though it
+can be empty.
+
 ### <a name="usage-instrumentation"></a> Instrumentation
 
 Some operations are instrumented using [Active Support Notifications](http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html).

data/config/phobos.yml.example

@@ -1,6 +1,9 @@
 logger:
   # Optional log file, set to false or remove to disable it
   file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
   level: info
   # Comment the block to disable ruby-kafka logs
   ruby_kafka:
@@ -95,6 +98,16 @@ listeners:
     # Apply this encoding to the message payload, if blank it uses the original encoding. This property accepts values
     # defined by the ruby Encoding class (https://ruby-doc.org/core-2.3.0/Encoding.html). Ex: UTF_8, ASCII_8BIT, etc
     force_encoding:
+    # Specify the delivery method for a listener.
+    # Possible values: [`message`, `batch` (default)]
+    #  - `message` will yield individual messages from Ruby Kafka using `each_message` and will commit/heartbeat at every consumed message.
+    #     This is overall a bit slower than using batch, but easier to configure.
+    #  - `batch` will yield batches from Ruby Kafka using `each_batch`, and commit/heartbeat at every consumed batch.
+    #     Due to implementation in Ruby Kafka, it should be noted that when configuring large batch sizes in combination with taking long time to consume messages,
+    #     your consumers might get kicked from the Kafka cluster for not sending a heartbeat within the expected interval.
+    #     Take this into consideration when determining your configuration.
+    # Note: Ultimately commit/heartbeart will depend on the offset commit options and the heartbeat interval.
+    delivery: batch
     # Use this if custom backoff is required for a listener
     backoff:
       min_ms: 500

data/docker-compose.yml

@@ -1,5 +1,17 @@
 version: '2'
 services:
+  test:
+    depends_on:
+      - kafka
+    build:
+      context: .
+    network_mode: service:kafka
+    environment:
+      - "DEFAULT_TIMEOUT=${DEFAULT_TIMEOUT}"
+    command: rspec
+    volumes:
+      - ./coverage:/opt/phobos/coverage
+
   zookeeper:
     image: jplock/zookeeper:3.4.10
     ports:

data/lib/phobos.rb

@@ -35,10 +35,16 @@ module Phobos
       @config = DeepStruct.new(fetch_settings(configuration))
       @config.class.send(:define_method, :producer_hash) { Phobos.config.producer&.to_hash }
       @config.class.send(:define_method, :consumer_hash) { Phobos.config.consumer&.to_hash }
+      @config.listeners ||= []
       configure_logger
       logger.info { Hash(message: 'Phobos configured', env: ENV['RACK_ENV']) }
     end
 
+    def add_listeners(listeners_configuration)
+      listeners_config = DeepStruct.new(fetch_settings(listeners_configuration))
+      @config.listeners += listeners_config.listeners
+    end
+
     def create_kafka_client
       Kafka.new(config.kafka.to_hash.merge(logger: @ruby_kafka_logger))
     end
@@ -54,8 +60,14 @@ module Phobos
       log_file = config.logger.file
       ruby_kafka = config.logger.ruby_kafka
       date_pattern = '%Y-%m-%dT%H:%M:%S:%L%zZ'
-      file_layout = Logging.layouts.json(date_pattern: date_pattern)
-      stdout_layout = Logging.layouts.pattern(date_pattern: date_pattern)
+      json_layout = Logging.layouts.json(date_pattern: date_pattern)
+
+      stdout_layout = if config.logger.stdout_json == true
+        json_layout
+      else
+        Logging.layouts.pattern(date_pattern: date_pattern)
+      end
+
       appenders = [Logging.appenders.stdout(layout: stdout_layout)]
 
       Logging.backtrace(true)
@@ -63,7 +75,7 @@ module Phobos
 
       if log_file
         FileUtils.mkdir_p(File.dirname(log_file))
-        appenders << Logging.appenders.file(log_file, layout: file_layout)
+        appenders << Logging.appenders.file(log_file, layout: json_layout)
       end
 
       @ruby_kafka_logger = nil

data/lib/phobos/actions/process_batch.rb

@@ -3,57 +3,32 @@ module Phobos
     class ProcessBatch
       include Phobos::Instrumentation
 
+      attr_reader :metadata
+
       def initialize(listener:, batch:, listener_metadata:)
         @listener = listener
         @batch = batch
         @listener_metadata = listener_metadata
+        @metadata = listener_metadata.merge(
+          batch_size: batch.messages.count,
+          partition: batch.partition,
+          offset_lag: batch.offset_lag
+        )
       end
 
       def execute
-        @batch.messages.each do |message|
-          backoff = @listener.create_exponential_backoff
-          metadata = @listener_metadata.merge(
-            key: message.key,
-            partition: message.partition,
-            offset: message.offset,
-            retry_count: 0
-          )
-
-          begin
-            instrument('listener.process_message', metadata) do |metadata|
-              time_elapsed = measure do
-                Phobos::Actions::ProcessMessage.new(
-                  listener: @listener,
-                  message: message,
-                  metadata: metadata,
-                  encoding: @listener.encoding
-                ).execute
-              end
-              metadata.merge!(time_elapsed: time_elapsed)
-            end
-          rescue => e
-            retry_count = metadata[:retry_count]
-            interval = backoff.interval_at(retry_count).round(2)
-
-            error = {
-              waiting_time: interval,
-              exception_class: e.class.name,
-              exception_message: e.message,
-              backtrace: e.backtrace
-            }
-
-            instrument('listener.retry_handler_error', error.merge(metadata)) do
-              Phobos.logger.error do
-                { message: "error processing message, waiting #{interval}s" }.merge(error).merge(metadata)
-              end
-
-              sleep interval
-              metadata.merge!(retry_count: retry_count + 1)
+        instrument('listener.process_batch', @metadata) do |metadata|
+          time_elapsed = measure do
+            @batch.messages.each do |message|
+              Phobos::Actions::ProcessMessage.new(
+                listener: @listener,
+                message: message,
+                listener_metadata: @listener_metadata
+              ).execute
             end
-
-            raise Phobos::AbortError if @listener.should_stop?
-            retry
           end
+
+          metadata.merge!(time_elapsed: time_elapsed)
         end
       end
     end

data/lib/phobos/actions/process_message.rb

@@ -1,25 +1,73 @@
 module Phobos
   module Actions
     class ProcessMessage
-      def initialize(listener:, message:, metadata:, encoding:)
+      include Phobos::Instrumentation
+
+      attr_reader :metadata
+
+      def initialize(listener:, message:, listener_metadata:)
         @listener = listener
         @message = message
-        @metadata = metadata
-        @encoding = encoding
+        @listener_metadata = listener_metadata
+        @metadata = listener_metadata.merge(
+          key: message.key,
+          partition: message.partition,
+          offset: message.offset,
+          retry_count: 0
+        )
       end
 
       def execute
+        backoff = @listener.create_exponential_backoff
         payload = force_encoding(@message.value)
-        decoded_payload = @listener.handler_class.new.before_consume(payload)
-        @listener.handler_class.around_consume(decoded_payload, @metadata) do
-          @listener.handler_class.new.consume(decoded_payload, @metadata)
+
+        begin
+          process_message(payload)
+        rescue => e
+          retry_count = @metadata[:retry_count]
+          interval = backoff.interval_at(retry_count).round(2)
+
+          error = {
+            waiting_time: interval,
+            exception_class: e.class.name,
+            exception_message: e.message,
+            backtrace: e.backtrace
+          }
+
+          instrument('listener.retry_handler_error', error.merge(@metadata)) do
+            Phobos.logger.error do
+              { message: "error processing message, waiting #{interval}s" }.merge(error).merge(@metadata)
+            end
+
+            sleep interval
+          end
+
+          raise Phobos::AbortError if @listener.should_stop?
+
+          @metadata.merge!(retry_count: retry_count + 1)
+          retry
         end
       end
 
       private
 
       def force_encoding(value)
-        @encoding ? value.force_encoding(@encoding) : value
+        @listener.encoding ? value.force_encoding(@listener.encoding) : value
+      end
+
+      def process_message(payload)
+        instrument('listener.process_message', @metadata) do |metadata|
+          time_elapsed = measure do
+            handler = @listener.handler_class.new
+            preprocessed_payload = handler.before_consume(payload)
+
+            @listener.handler_class.around_consume(preprocessed_payload, @metadata) do
+              handler.consume(preprocessed_payload, @metadata)
+            end
+          end
+
+          metadata.merge!(time_elapsed: time_elapsed)
+        end
       end
     end
   end

data/lib/phobos/cli.rb

@@ -49,6 +49,13 @@ module Phobos
              aliases: ['-b'],
              banner: 'File path to load application specific code',
              default: 'phobos_boot.rb'
+      option :listeners,
+             aliases: ['-l'],
+             banner: 'Separate listeners config file (optional)'
+      option :skip_config,
+             default: false,
+             type: :boolean,
+             banner: 'Skip config file'
       def start
         Phobos::CLI::Start.new(options).execute
       end

data/lib/phobos/cli/start.rb

@@ -4,14 +4,28 @@ module Phobos
   module CLI
     class Start
       def initialize(options)
-        @config_file = File.expand_path(options[:config])
+        unless options[:skip_config]
+          @config_file = File.expand_path(options[:config])
+        end
         @boot_file = File.expand_path(options[:boot])
+
+        if options[:listeners]
+          @listeners_file = File.expand_path(options[:listeners])
+        end
       end
 
       def execute
-        validate_config_file!
-        Phobos.configure(config_file)
+        if config_file
+          validate_config_file!
+          Phobos.configure(config_file)
+        end
+
         load_boot_file
+
+        if listeners_file
+          Phobos.add_listeners(listeners_file)
+        end
+
         validate_listeners!
 
         Phobos::CLI::Runner.new.run!
@@ -19,7 +33,7 @@ module Phobos
 
       private
 
-      attr_reader :config_file, :boot_file
+      attr_reader :config_file, :boot_file, :listeners_file
 
       def validate_config_file!
         unless File.exist?(config_file)
@@ -29,13 +43,27 @@ module Phobos
       end
 
       def validate_listeners!
-        Phobos.config.listeners.collect(&:handler).each do |handler_class|
+        Phobos.config.listeners.each do |listener|
+          handler_class = listener.handler
+
           begin
             handler_class.constantize
           rescue NameError
             Phobos::CLI.logger.error { Hash(message: "Handler '#{handler_class}' not defined") }
             exit(1)
           end
+
+          delivery = listener.delivery
+          if delivery.nil?
+            Phobos::CLI.logger.warn do
+              Hash(message: "Delivery option should be specified, defaulting to 'batch' - specify this option to silence this message")
+            end
+          elsif !Listener::DELIVERY_OPTS.include?(delivery)
+            Phobos::CLI.logger.error do
+              Hash(message: "Invalid delivery option '#{delivery}'. Please specify one of: #{Listener::DELIVERY_OPTS.join(', ')}")
+            end
+            exit(1)
+          end
         end
       end
 

data/lib/phobos/executor.rb

@@ -11,6 +11,7 @@ module Phobos
       start_from_beginning
       max_bytes_per_partition
       backoff
+      delivery
     ).freeze
 
     def initialize

data/lib/phobos/listener.rb

@@ -4,6 +4,7 @@ module Phobos
 
     KAFKA_CONSUMER_OPTS = %i(session_timeout offset_commit_interval offset_commit_threshold heartbeat_interval).freeze
     DEFAULT_MAX_BYTES_PER_PARTITION = 524288 # 512 KB
+    DELIVERY_OPTS = %w[batch message].freeze
 
     attr_reader :group_id, :topic, :id
     attr_reader :handler_class, :encoding
@@ -11,12 +12,14 @@ module Phobos
     def initialize(handler:, group_id:, topic:, min_bytes: nil,
                    max_wait_time: nil, force_encoding: nil,
                    start_from_beginning: true, backoff: nil,
+                   delivery: 'batch',
                    max_bytes_per_partition: DEFAULT_MAX_BYTES_PER_PARTITION)
       @id = SecureRandom.hex[0...6]
       @handler_class = handler
       @group_id = group_id
       @topic = topic
       @backoff = backoff
+      @delivery = delivery.to_s
       @subscribe_opts = {
         start_from_beginning: start_from_beginning,
         max_bytes_per_partition: max_bytes_per_partition
@@ -42,29 +45,7 @@ module Phobos
       end
 
       begin
-        @consumer.each_batch(@consumer_opts) do |batch|
-          batch_metadata = {
-            batch_size: batch.messages.count,
-            partition: batch.partition,
-            offset_lag: batch.offset_lag,
-            # the offset of the most recent message in the partition
-            highwater_mark_offset: batch.highwater_mark_offset
-          }.merge(listener_metadata)
-
-          instrument('listener.process_batch', batch_metadata) do |batch_metadata|
-            time_elapsed = measure do
-              Phobos::Actions::ProcessBatch.new(
-                listener: self,
-                batch: batch,
-                listener_metadata: listener_metadata
-              ).execute
-            end
-            batch_metadata.merge!(time_elapsed: time_elapsed)
-            Phobos.logger.info { Hash(message: 'Committed offset').merge(batch_metadata) }
-          end
-
-          return if should_stop?
-        end
+        @delivery == 'batch' ? consume_each_batch : consume_each_message
 
       # Abort is an exception to prevent the consumer from committing the offset.
       # Since "listener" had a message being retried while "stop" was called
@@ -95,6 +76,34 @@ module Phobos
       end
     end
 
+    def consume_each_batch
+      @consumer.each_batch(@consumer_opts) do |batch|
+        batch_processor = Phobos::Actions::ProcessBatch.new(
+          listener: self,
+          batch: batch,
+          listener_metadata: listener_metadata
+        )
+
+        batch_processor.execute
+        Phobos.logger.info { Hash(message: 'Committed offset').merge(batch_processor.metadata) }
+        return if should_stop?
+      end
+    end
+
+    def consume_each_message
+      @consumer.each_message(@consumer_opts) do |message|
+        message_processor = Phobos::Actions::ProcessMessage.new(
+          listener: self,
+          message: message,
+          listener_metadata: listener_metadata
+        )
+
+        message_processor.execute
+        Phobos.logger.info { Hash(message: 'Committed offset').merge(message_processor.metadata) }
+        return if should_stop?
+      end
+    end
+
     def stop
       return if should_stop?
       instrument('listener.stopping', listener_metadata) do

data/lib/phobos/version.rb

@@ -1,3 +1,3 @@
 module Phobos
-  VERSION = '1.5.0'
+  VERSION = '1.6.0'
 end

data/phobos.gemspec

@@ -44,16 +44,14 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
   spec.required_ruby_version = '>= 2.3'
 
-  spec.add_development_dependency 'bundler', '~> 1.12'
+  spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.6'
-  spec.add_development_dependency 'pry-byebug', '~> 3.4.3'
-  spec.add_development_dependency 'rspec_junit_formatter', '0.3.0'
-  spec.add_development_dependency 'simplecov', '~> 0.14.1'
-  spec.add_development_dependency 'coveralls', '~> 0.8.21'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'timecop'
 
-  spec.add_dependency 'ruby-kafka', '>= 0.3.14'
+  spec.add_dependency 'ruby-kafka', '>= 0.3.14', '< 0.5.0'
   spec.add_dependency 'concurrent-ruby', '>= 1.0.2'
   spec.add_dependency 'concurrent-ruby-ext', '>= 1.0.2'
   spec.add_dependency 'activesupport', '>= 4.0.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -14,22 +14,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-26 00:00:00.000000000 Z
+date: 2017-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -50,70 +50,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.4.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.4.3
-- !ruby/object:Gem::Dependency
-  name: rspec_junit_formatter
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.14.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.14.1
-- !ruby/object:Gem::Dependency
-  name: coveralls
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.21
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.21
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -135,6 +107,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.14
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -142,6 +117,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.14
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -242,9 +220,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".dockerignore"
+- ".env"
 - ".gitignore"
 - ".rspec"
 - ".ruby-version"
+- ".travis.yml"
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
@@ -254,7 +235,6 @@ files:
 - bin/console
 - bin/phobos
 - bin/setup
-- circle.yml
 - config/phobos.yml.example
 - docker-compose.yml
 - examples/handler_saving_events_database.rb

data/circle.yml

@@ -1,27 +0,0 @@
-machine:
-  pre:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
-  services:
-    - docker
-  environment:
-    LOG_LEVEL: DEBUG
-    CI: true
-    DEFAULT_TIMEOUT: 20
-  ruby:
-    version: 2.4.1
-
-dependencies:
-  pre:
-    - docker -v
-    - docker pull ches/kafka:0.10.2.1
-    - docker pull jplock/zookeeper:3.4.10
-    - gem install bundler -v 1.13.2
-    - bundle install
-
-test:
-  override:
-    - docker run -d -p 2003:2181 --name zookeeper jplock/zookeeper:3.4.10; sleep 5
-    - docker run -d -p 9092:9092 --name kafka -e KAFKA_BROKER_ID=0 -e KAFKA_ADVERTISED_HOST_NAME=localhost -e KAFKA_ADVERTISED_PORT=9092 -e ZOOKEEPER_CONNECTION_STRING=zookeeper:2181 --link zookeeper:zookeeper ches/kafka:0.10.2.1; sleep 5
-    - bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/unit.xml
-  post:
-    - cp log/*.log $CIRCLE_ARTIFACTS/ || true