phobos 2.1.1 → 2.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ec70d6f78852088846db2f5cee6673e9889962379bd82934758aa936886f1df
-  data.tar.gz: f0d5a40fb1acbf02e0ac6cae4121571eb72689bc13076505153f33cef48f2362
+  metadata.gz: 02e1b97b3d44dffefffea99da0c15393997043598697d88fe3f7c7ab83549193
+  data.tar.gz: 4e3ec45a1db997825ace585a1535ac5c1504105afabe0470c0c676f8f301c66a
 SHA512:
-  metadata.gz: c7192c98b0e3f3b22ab3aa4ef4f28bf3227410120012d3b77a66f018960f0e72309f72d47a00e505355d906826dc89bdc327ceea50a99ea7e28d233602bd8ec6
-  data.tar.gz: b1dab967fb374559b98d976b0628140692f9c080d3de55d17edaa2a885a4ef17a1bbf7555a380ef00199e62a002f0bdd6d56486e77fbe4f2176eb39756c4f185
+  metadata.gz: b249c05f44dfe0dcb0f81b8177dd27a129e4364e0dbcbf8492295d9823473a7de66ee913c088945346a4e1943b3f2226c8106083cf5502d90341b8bb8fbac0eb
+  data.tar.gz: f6725f584a75f099bb2b4624338e424f7ba44859ace5c660bfc20d8f5681d663a1cd051381d18f832659f87bd1406d5c8aea3098112876a1f119649b56645be6

data/CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ``
 ## UNRELEASED
 
+## [2.1.4] - 2022-06-15
+
+- Re-raise consuming errors so that threads don't die forever and are retried
+
+## [2.1.3] - 2022-05-13
+
+- Rescue and log Kafka::HeartbeatError in batch processing
+
+## [2.1.2] - 2022-01-27
+
+- Fix packaging error.
+
 ## [2.1.1] - 2022-01-26
 
 - Fix crash on Ruby 3.1 involving a change to the Psych gem.

data/lib/phobos/actions/process_batch.rb

@@ -4,6 +4,7 @@ module Phobos
   module Actions
     class ProcessBatch
       include Phobos::Instrumentation
+      include Phobos::Log
 
       attr_reader :metadata
 
@@ -26,7 +27,11 @@ module Phobos
               message: message,
               listener_metadata: @listener_metadata
             ).execute
-            @listener.consumer.trigger_heartbeat
+            begin
+              @listener.consumer.trigger_heartbeat
+            rescue Kafka::HeartbeatError => e
+              log_warn("Error sending Heartbeat #{e.class.name}-#{e}")
+            end
           end
         end
       end

data/lib/phobos/configuration.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'phobos/deep_struct'
+
+module Phobos
+  module Configuration
+    def configure(configuration)
+      @config = fetch_configuration(configuration)
+      @config.class.send(:define_method, :producer_hash) do
+        Phobos.config.producer&.to_hash&.except(:kafka)
+      end
+      @config.class.send(:define_method, :consumer_hash) do
+        Phobos.config.consumer&.to_hash&.except(:kafka)
+      end
+      @config.listeners ||= []
+      configure_logger
+    end
+
+    # :nodoc:
+    def configure_logger
+      Logging.backtrace(true)
+      Logging.logger.root.level = silence_log ? :fatal : config.logger.level
+
+      configure_ruby_kafka_logger
+      configure_phobos_logger
+
+      logger.info do
+        Hash(message: 'Phobos configured', env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'N/A')
+      end
+    end
+
+    private
+
+    def fetch_configuration(configuration)
+      DeepStruct.new(read_configuration(configuration))
+    end
+
+    def read_configuration(configuration)
+      return configuration.to_h if configuration.respond_to?(:to_h)
+
+      config_erb = ERB.new(File.read(File.expand_path(configuration))).result
+
+      YAML.safe_load(
+        config_erb,
+        permitted_classes: [Symbol],
+        permitted_symbols: [],
+        aliases: true
+      )
+    rescue ArgumentError
+      YAML.safe_load(config_erb, [Symbol], [], true)
+    end
+
+    def configure_phobos_logger
+      if config.custom_logger
+        @logger = config.custom_logger
+      else
+        @logger = Logging.logger[self]
+        @logger.appenders = logger_appenders
+      end
+    end
+
+    def configure_ruby_kafka_logger
+      if config.custom_kafka_logger
+        @ruby_kafka_logger = config.custom_kafka_logger
+      elsif config.logger.ruby_kafka
+        @ruby_kafka_logger = Logging.logger['RubyKafka']
+        @ruby_kafka_logger.appenders = logger_appenders
+        @ruby_kafka_logger.level = silence_log ? :fatal : config.logger.ruby_kafka.level
+      else
+        @ruby_kafka_logger = nil
+      end
+    end
+
+    def logger_appenders
+      appenders = [Logging.appenders.stdout(layout: stdout_layout)]
+
+      if log_file
+        FileUtils.mkdir_p(File.dirname(log_file))
+        appenders << Logging.appenders.file(log_file, layout: json_layout)
+      end
+
+      appenders
+    end
+
+    def log_file
+      config.logger.file
+    end
+
+    def json_layout
+      Logging.layouts.json(date_pattern: Constants::LOG_DATE_PATTERN)
+    end
+
+    def stdout_layout
+      if config.logger.stdout_json == true
+        json_layout
+      else
+        Logging.layouts.pattern(date_pattern: Constants::LOG_DATE_PATTERN)
+      end
+    end
+  end
+end

data/lib/phobos/listener.rb

@@ -42,12 +42,11 @@ module Phobos
 
     def start
       @signal_to_stop = false
-
       start_listener
 
       begin
         start_consumer_loop
-      rescue Kafka::ProcessingError, Phobos::AbortError
+      rescue Kafka::ProcessingError, Phobos::AbortError => e
         # Abort is an exception to prevent the consumer from committing the offset.
         # Since "listener" had a message being retried while "stop" was called
         # it's wise to not commit the batch offset to avoid data loss. This will
@@ -55,6 +54,7 @@ module Phobos
         instrument('listener.retry_aborted', listener_metadata) do
           log_info('Retry loop aborted, listener is shutting down', listener_metadata)
         end
+        raise e if e.is_a?(Kafka::ProcessingError)
       end
     ensure
       stop_listener

data/lib/phobos/log.rb

@@ -13,6 +13,10 @@ module Phobos
     def log_error(msg, metadata)
       LoggerHelper.log(:error, msg, metadata)
     end
+
+    def log_warn(msg, metadata = {})
+      LoggerHelper.log(:warn, msg, metadata)
+    end
   end
 
   module LoggerHelper

data/lib/phobos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phobos
-  VERSION = '2.1.1'
+  VERSION = '2.1.4'
 end

data/lib/phobos.rb

@@ -80,6 +80,5 @@ module Phobos
       location = caller.find { |line| line !~ %r{/phobos/} }
       warn "DEPRECATION WARNING: #{message}: #{location}"
     end
-
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.4
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -15,7 +15,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-26 00:00:00.000000000 Z
+date: 2022-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -277,6 +277,7 @@ files:
 - lib/phobos/cli.rb
 - lib/phobos/cli/runner.rb
 - lib/phobos/cli/start.rb
+- lib/phobos/configuration.rb
 - lib/phobos/constants.rb
 - lib/phobos/deep_struct.rb
 - lib/phobos/echo_handler.rb
@@ -314,7 +315,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.5
 signing_key:
 specification_version: 4
 summary: Simplifying Kafka for ruby apps