phobos 1.6.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e7de0ef712ba1636a2c66ceb7fbfa159e479c87
-  data.tar.gz: 5bbaa97f07e0f6aa5825b49220daa2f6cac239dd
+  metadata.gz: aa6ceb9067bc80835a1460e43cbe613f7fdaaa23
+  data.tar.gz: 43281f6ada2fe5b824da9cc236d56523c58cee02
 SHA512:
-  metadata.gz: 3ffa62ee7edcd651dabc33fa6f3bfc96cf3ab30076b49254d89e1c6f0443843159022945c483126fa3e0d5ac8674526a350693405448b327681d0752dc281b31
-  data.tar.gz: aaf894ba3a7ac6c5c01d1b06ee621995fecd5bd0ed368bc28235bdd2a756b55fcb9d7f8955987429e8ed3ae850dc231c4d43ef6fb08e5c611685caf4929729bb
+  metadata.gz: ad5569add3dd9748cb746943816285d3518052dbc148b2102975b60744782081afc81c5d520f7ca80e090505b4c6b592b49959aa8d22820367f799739d4c2e7d
+  data.tar.gz: 2e110cdba90db10a6c0cf4a51bee1f8bdf31ea1090f303d44d817654d6c8dd14837eba52a6a83419e35c31a385a3dc01a083df16142128805adc9e5d5d65b29c

data/CHANGELOG.md

@@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## UNRELEASED
 
+## [1.7.0] - 2017-12-05
+### Fixed
+- Test are failing with ruby-kafka 0.5.0 #48
+- Allow Phobos to run in apps using ActiveSupport 3.x #57
+### Added
+- Property (handler) added to listener instrumentation #60
+### Removed
+- Property (time_elapsed) removed - use duration instead #24
+### Changed
+- Max bytes per partition is now 1 MB by default #56
+
 ## [1.6.1] - 2017-11-16
 ### Fixed
 - `Phobos::Test::Helper` is broken #53

data/README.md

@@ -378,32 +378,35 @@ end
     * listener_id
     * group_id
     * topic
+    * handler
   * `listener.start` is sent when listener starts. It includes the following payload:
     * listener_id
     * group_id
     * topic
+    * handler
   * `listener.process_batch` is sent after process a batch. It includes the following payload:
     * listener_id
     * group_id
     * topic
+    * handler
     * batch_size
     * partition
     * offset_lag
     * highwater_mark_offset
-    * time_elapsed
   * `listener.process_message` is sent after process a message. It includes the following payload:
     * listener_id
     * group_id
     * topic
+    * handler
     * key
     * partition
     * offset
     * retry_count
-    * time_elapsed
   * `listener.retry_handler_error` is sent after waited for `handler#consume` retry. It includes the following payload:
     * listener_id
     * group_id
     * topic
+    * handler
     * key
     * partition
     * offset
@@ -416,18 +419,22 @@ end
     * listener_id
     * group_id
     * topic
+    * handler
   * `listener.stopping` is sent when the listener receives signal to stop
     * listener_id
     * group_id
     * topic
+    * handler
   * `listener.stop_handler` is sent after stopping the handler
     * listener_id
     * group_id
     * topic
+    * handler
   * `listener.stop` is send after stopping the listener
     * listener_id
     * group_id
     * topic
+    * handler
 
 ## <a name="plugins"></a> Plugins
 
@@ -438,6 +445,15 @@ List of gems that enhance Phobos:
   * Ensures that your handler will consume messages only once
   * Allows your system to quickly reprocess events in case of failures
 
+* [Phobos Checkpoint UI](https://github.com/klarna/phobos_checkpoint_ui) gives your Phobos DB Checkpoint powered app a web gui with the features below. Maintaining a Kafka consumer app has never been smoother:
+  * Search events and inspect payload
+  * See failures and retry / delete them
+
+* [Phobos Prometheus](https://github.com/phobos/phobos_prometheus) adds prometheus metrics to your phobos consumer.
+  * Measures total messages and batches processed
+  * Measures total duration needed to process each message (and batch)
+  * Adds `/metrics` endpoit to scrape data
+
 ## <a name="development"></a> Development
 
 After checking out the repo:

data/lib/phobos/actions/process_batch.rb

@@ -18,17 +18,13 @@ module Phobos
 
       def execute
         instrument('listener.process_batch', @metadata) do |metadata|
-          time_elapsed = measure do
-            @batch.messages.each do |message|
-              Phobos::Actions::ProcessMessage.new(
-                listener: @listener,
-                message: message,
-                listener_metadata: @listener_metadata
-              ).execute
-            end
+          @batch.messages.each do |message|
+            Phobos::Actions::ProcessMessage.new(
+              listener: @listener,
+              message: message,
+              listener_metadata: @listener_metadata
+            ).execute
           end
-
-          metadata.merge!(time_elapsed: time_elapsed)
         end
       end
     end

data/lib/phobos/actions/process_message.rb

@@ -8,7 +8,6 @@ module Phobos
       def initialize(listener:, message:, listener_metadata:)
         @listener = listener
         @message = message
-        @listener_metadata = listener_metadata
         @metadata = listener_metadata.merge(
           key: message.key,
           partition: message.partition,
@@ -56,20 +55,13 @@ module Phobos
       end
 
       def process_message(payload)
-        instrument('listener.process_message', @metadata) do |metadata|
-          consume_result = nil
-          time_elapsed = measure do
-            handler = @listener.handler_class.new
-            preprocessed_payload = handler.before_consume(payload)
+        instrument('listener.process_message', @metadata) do
+          handler = @listener.handler_class.new
+          preprocessed_payload = handler.before_consume(payload)
 
-            @listener.handler_class.around_consume(preprocessed_payload, @metadata) do
-              consume_result = handler.consume(preprocessed_payload, @metadata)
-            end
+          @listener.handler_class.around_consume(preprocessed_payload, @metadata) do
+            handler.consume(preprocessed_payload, @metadata)
           end
-
-          metadata.merge!(time_elapsed: time_elapsed)
-
-          consume_result
         end
       end
     end

data/lib/phobos/cli.rb

@@ -41,21 +41,21 @@ module Phobos
       end
 
       desc 'start', 'Starts Phobos'
-      option :config,
-             aliases: ['-c'],
-             default: 'config/phobos.yml',
-             banner: 'Configuration file'
-      option :boot,
-             aliases: ['-b'],
-             banner: 'File path to load application specific code',
-             default: 'phobos_boot.rb'
-      option :listeners,
-             aliases: ['-l'],
-             banner: 'Separate listeners config file (optional)'
-      option :skip_config,
-             default: false,
-             type: :boolean,
-             banner: 'Skip config file'
+      method_option :config,
+                    aliases: ['-c'],
+                    default: 'config/phobos.yml',
+                    banner: 'Configuration file'
+      method_option :boot,
+                    aliases: ['-b'],
+                    banner: 'File path to load application specific code',
+                    default: 'phobos_boot.rb'
+      method_option :listeners,
+                    aliases: ['-l'],
+                    banner: 'Separate listeners config file (optional)'
+      method_option :skip_config,
+                    default: false,
+                    type: :boolean,
+                    banner: 'Skip config file'
       def start
         Phobos::CLI::Start.new(options).execute
       end

data/lib/phobos/instrumentation.rb

@@ -1,3 +1,5 @@
+require 'active_support/notifications'
+
 module Phobos
   module Instrumentation
     NAMESPACE = 'phobos'
@@ -17,11 +19,5 @@ module Phobos
         yield(extra) if block_given?
       end
     end
-
-    def measure
-      start = Time.now.utc
-      yield if block_given?
-      (Time.now.utc - start).round(3)
-    end
   end
 end

data/lib/phobos/listener.rb

@@ -3,7 +3,7 @@ module Phobos
     include Phobos::Instrumentation
 
     KAFKA_CONSUMER_OPTS = %i(session_timeout offset_commit_interval offset_commit_threshold heartbeat_interval).freeze
-    DEFAULT_MAX_BYTES_PER_PARTITION = 524288 # 512 KB
+    DEFAULT_MAX_BYTES_PER_PARTITION = 1048576 # 1 MB
     DELIVERY_OPTS = %w[batch message].freeze
 
     attr_reader :group_id, :topic, :id
@@ -124,7 +124,7 @@ module Phobos
     private
 
     def listener_metadata
-      { listener_id: id, group_id: group_id, topic: topic }
+      { listener_id: id, group_id: group_id, topic: topic, handler: handler_class.to_s }
     end
 
     def create_kafka_consumer

data/lib/phobos/version.rb

@@ -1,3 +1,3 @@
 module Phobos
-  VERSION = '1.6.1'
+  VERSION = '1.7.0'
 end

data/phobos.gemspec

@@ -51,10 +51,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'timecop'
 
-  spec.add_dependency 'ruby-kafka', '>= 0.3.14', '< 0.5.0'
+  spec.add_dependency 'ruby-kafka', '>= 0.3.14'
   spec.add_dependency 'concurrent-ruby', '>= 1.0.2'
   spec.add_dependency 'concurrent-ruby-ext', '>= 1.0.2'
-  spec.add_dependency 'activesupport', '>= 4.0.0'
+  spec.add_dependency 'activesupport', '>= 3.0.0'
   spec.add_dependency 'logging'
   spec.add_dependency 'exponential-backoff'
   spec.add_dependency 'thor'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2017-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,9 +107,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.14
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -117,9 +114,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.14
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -154,14 +148,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: logging
   requirement: !ruby/object:Gem::Requirement