phobos 2.0.0.pre.beta1 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 646f966d9d307346090b88a3ef52f784da42f07a7f8fd3369131a4d1149fc293
-  data.tar.gz: 3631aef66014166e318f24b8b282265e0d3663ed006b915c49762301e8f02b89
+  metadata.gz: 6ec70d6f78852088846db2f5cee6673e9889962379bd82934758aa936886f1df
+  data.tar.gz: f0d5a40fb1acbf02e0ac6cae4121571eb72689bc13076505153f33cef48f2362
 SHA512:
-  metadata.gz: c19eedfd196604f55ea659d501323cecfca50c4c53a03ae1193457f344e4a1e580bbd7ca2b33559b20ab2c1418143b48b6d9e4942986f71b369f4a9e261ccfeb
-  data.tar.gz: cdbd2146b341d91bb7131eb986e24055bf73054ed12eb542e09c9d2a56b4b462bd4d3015e5ee3a00934e4e125658e5438987b402aa2edb0ca2fac2172b098f58
+  metadata.gz: c7192c98b0e3f3b22ab3aa4ef4f28bf3227410120012d3b77a66f018960f0e72309f72d47a00e505355d906826dc89bdc327ceea50a99ea7e28d233602bd8ec6
+  data.tar.gz: b1dab967fb374559b98d976b0628140692f9c080d3de55d17edaa2a885a4ef17a1bbf7555a380ef00199e62a002f0bdd6d56486e77fbe4f2176eb39756c4f185

data/.gitignore

@@ -12,3 +12,5 @@ spec/examples.txt
 config/*.yml
 log/*.log
 .byebug_history
+.idea
+*.gem

data/CHANGELOG.md

@@ -6,7 +6,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ``
 ## UNRELEASED
 
-## [2.0.0-beta1] - 2020-5-04
+## [2.1.1] - 2022-01-26
+
+- Fix crash on Ruby 3.1 involving a change to the Psych gem.
+
+## [2.1.0] - 2021-05-27
+
+- Modify config to allow specifying kafka connection information separately for consumers and producers
+
+## [2.0.2] - 2021-01-28
+- Additional fixes for Ruby 3.0
+
+## [2.0.1] - 2021-01-14
+- Fix bug with Ruby 3.0 around OpenStruct
+
+## [2.0.0-beta1] - 2020-05-04
 
 - Remove deprecated patterns:
 -- `before_consume` method

data/README.md

@@ -404,9 +404,12 @@ All [options supported by  `ruby-kafka`][ruby-kafka-client] can be provided.
 __producer__ provides configurations for all producers created over the application,
 the options are the same for regular and async producers.
 All [options supported by  `ruby-kafka`][ruby-kafka-producer] can be provided.
+If the __kafka__ key is present under __producer__, it is merged into the top-level __kafka__, allowing different connection configuration for producers.
 
 __consumer__ provides configurations for all consumer groups created over the application.
 All [options supported by  `ruby-kafka`][ruby-kafka-consumer] can be provided.
+If the __kafka__ key is present under __consumer__, it is merged into the top-level __kafka__, allowing different connection configuration for consumers.
+
 
 __backoff__ Phobos provides automatic retries for your handlers. If an exception
 is raised, the listener will retry following the back off configured here.
@@ -582,8 +585,11 @@ After checking out the repo:
 * make sure `docker` is installed and running (for windows and mac this also includes `docker-compose`).
 * Linux: make sure `docker-compose` is installed and running.
 * run `bin/setup` to install dependencies
-* run `docker-compose up` to start the required kafka containers in a window
-* run `rspec` to run the tests in another window
+* run `docker-compose up -d --force-recreate kafka zookeeper` to start the required kafka containers
+* run tests to confirm no environmental issues
+  * wait a few seconds for kafka broker to get set up - `sleep 30`
+  * run `docker-compose run --rm test`
+  * make sure it reports `X examples, 0 failures`
 
 You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
@@ -596,8 +602,16 @@ Phobos exports a spec helper that can help you test your consumer. The Phobos li
 * `process_message(handler:, payload:, metadata: {}, encoding: nil)` - Invokes your handler with payload and metadata, using a dummy listener (encoding and metadata are optional).
 
 ```ruby
-require 'spec_helper'
+### spec_helper.rb
+require 'phobos/test/helper'
+RSpec.configure do |config|
+  config.include Phobos::Test::Helper
+  config.before(:each) do
+    Phobos.configure(path_to_my_config_file)
+  end
+end 
 
+### Spec file
 describe MyConsumer do
   let(:payload) { 'foo' }
   let(:metadata) { Hash(foo: 'bar') }

data/config/phobos.yml.example

@@ -65,6 +65,10 @@ producer:
   # that you need to manually call sync_producer_shutdown before exiting,
   # similar to async_producer_shutdown.
   persistent_connections: false
+  # kafka here supports the same parameters as the top-level, allowing custom connection
+  # configuration details for producers
+  kafka:
+    connect_timeout: 120
 
 consumer:
   # number of seconds after which, if a client hasn't contacted the Kafka cluster,
@@ -79,6 +83,10 @@ consumer:
   offset_retention_time:
   # interval between heartbeats; must be less than the session window
   heartbeat_interval: 10
+  # kafka here supports the same parameters as the top-level, allowing custom connection
+  # configuration details for consumers
+  kafka:
+    connect_timeout: 130
 
 backoff:
   min_ms: 1000

data/lib/phobos/deep_struct.rb

@@ -8,7 +8,7 @@ module Phobos
     # Based on
     # https://docs.omniref.com/ruby/2.3.0/files/lib/ostruct.rb#line=88
     def initialize(hash = nil)
-      @table = {}
+      super
       @hash_table = {}
 
       hash&.each_pair do |key, value|

data/lib/phobos/executor.rb

@@ -13,7 +13,7 @@ module Phobos
         max_concurrency = listener_configs[:max_concurrency] || 1
         Array.new(max_concurrency).map do
           configs = listener_configs.select { |k| Constants::LISTENER_OPTS.include?(k) }
-          Phobos::Listener.new(configs.merge(handler: handler_class))
+          Phobos::Listener.new(**configs.merge(handler: handler_class))
         end
       end
     end

data/lib/phobos/listener.rb

@@ -35,7 +35,7 @@ module Phobos
       )
       @encoding = Encoding.const_get(force_encoding.to_sym) if force_encoding
       @message_processing_opts = compact(min_bytes: min_bytes, max_wait_time: max_wait_time)
-      @kafka_client = Phobos.create_kafka_client
+      @kafka_client = Phobos.create_kafka_client(:consumer)
       @producer_enabled = @handler_class.ancestors.include?(Phobos::Producer)
     end
     # rubocop:enable Metrics/MethodLength
@@ -93,7 +93,7 @@ module Phobos
     def start_listener
       instrument('listener.start', listener_metadata) do
         @consumer = create_kafka_consumer
-        @consumer.subscribe(topic, @subscribe_opts)
+        @consumer.subscribe(topic, **@subscribe_opts)
 
         # This is done here because the producer client is bound to the current thread and
         # since "start" blocks a thread might be used to call it
@@ -135,7 +135,7 @@ module Phobos
     end
 
     def consume_each_batch
-      @consumer.each_batch(@message_processing_opts) do |batch|
+      @consumer.each_batch(**@message_processing_opts) do |batch|
         batch_processor = Phobos::Actions::ProcessBatch.new(
           listener: self,
           batch: batch,
@@ -149,7 +149,7 @@ module Phobos
     end
 
     def consume_each_batch_inline
-      @consumer.each_batch(@message_processing_opts) do |batch|
+      @consumer.each_batch(**@message_processing_opts) do |batch|
         batch_processor = Phobos::Actions::ProcessBatchInline.new(
           listener: self,
           batch: batch,
@@ -163,7 +163,7 @@ module Phobos
     end
 
     def consume_each_message
-      @consumer.each_message(@message_processing_opts) do |message|
+      @consumer.each_message(**@message_processing_opts) do |message|
         message_processor = Phobos::Actions::ProcessMessage.new(
           listener: self,
           message: message,
@@ -181,7 +181,7 @@ module Phobos
         Constants::KAFKA_CONSUMER_OPTS.include?(k)
       end
       configs.merge!(@kafka_consumer_opts)
-      @kafka_client.consumer({ group_id: group_id }.merge(configs))
+      @kafka_client.consumer(**{ group_id: group_id }.merge(configs))
     end
 
     def compact(hash)

data/lib/phobos/producer.rb

@@ -77,8 +77,8 @@ module Phobos
         end
 
         def create_sync_producer
-          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
-          sync_producer = client.producer(regular_configs)
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client(:producer))
+          sync_producer = client.producer(**regular_configs)
           if Phobos.config.producer_hash[:persistent_connections]
             producer_store[:sync_producer] = sync_producer
           end
@@ -108,8 +108,8 @@ module Phobos
         end
 
         def create_async_producer
-          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
-          async_producer = client.async_producer(async_configs)
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client(:producer))
+          async_producer = client.async_producer(**async_configs)
           producer_store[:async_producer] = async_producer
         end
 

data/lib/phobos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phobos
-  VERSION = '2.0.0-beta1'
+  VERSION = '2.1.1'
 end

data/lib/phobos.rb

@@ -13,7 +13,7 @@ require 'kafka'
 require 'logging'
 require 'erb'
 
-require 'phobos/deep_struct'
+require 'phobos/configuration'
 require 'phobos/version'
 require 'phobos/constants'
 require 'phobos/log'
@@ -49,25 +49,24 @@ module Logging
 end
 
 module Phobos
+  extend Configuration
   class << self
     attr_reader :config, :logger
     attr_accessor :silence_log
 
-    def configure(configuration)
-      @config = fetch_configuration(configuration)
-      @config.class.send(:define_method, :producer_hash) { Phobos.config.producer&.to_hash }
-      @config.class.send(:define_method, :consumer_hash) { Phobos.config.consumer&.to_hash }
-      @config.listeners ||= []
-      configure_logger
-    end
-
     def add_listeners(configuration)
       listeners_config = fetch_configuration(configuration)
       @config.listeners += listeners_config.listeners
     end
 
-    def create_kafka_client
-      Kafka.new(config.kafka.to_hash.merge(logger: @ruby_kafka_logger))
+    def create_kafka_client(config_key = nil)
+      kafka_config = config.kafka.to_hash.merge(logger: @ruby_kafka_logger)
+
+      if config_key
+        kafka_config = kafka_config.merge(**config.send(config_key)&.kafka&.to_hash || {})
+      end
+
+      Kafka.new(**kafka_config)
     end
 
     def create_exponential_backoff(backoff_config = nil)
@@ -82,84 +81,5 @@ module Phobos
       warn "DEPRECATION WARNING: #{message}: #{location}"
     end
 
-    # :nodoc:
-    def configure_logger
-      Logging.backtrace(true)
-      Logging.logger.root.level = silence_log ? :fatal : config.logger.level
-
-      configure_ruby_kafka_logger
-      configure_phobos_logger
-
-      logger.info do
-        Hash(message: 'Phobos configured', env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'N/A')
-      end
-    end
-
-    private
-
-    def fetch_configuration(configuration)
-      DeepStruct.new(read_configuration(configuration))
-    end
-
-    def read_configuration(configuration)
-      return configuration.to_h if configuration.respond_to?(:to_h)
-
-      YAML.safe_load(
-        ERB.new(
-          File.read(File.expand_path(configuration))
-        ).result,
-        [Symbol],
-        [],
-        true
-      )
-    end
-
-    def configure_phobos_logger
-      if config.custom_logger
-        @logger = config.custom_logger
-      else
-        @logger = Logging.logger[self]
-        @logger.appenders = logger_appenders
-      end
-    end
-
-    def configure_ruby_kafka_logger
-      if config.custom_kafka_logger
-        @ruby_kafka_logger = config.custom_kafka_logger
-      elsif config.logger.ruby_kafka
-        @ruby_kafka_logger = Logging.logger['RubyKafka']
-        @ruby_kafka_logger.appenders = logger_appenders
-        @ruby_kafka_logger.level = silence_log ? :fatal : config.logger.ruby_kafka.level
-      else
-        @ruby_kafka_logger = nil
-      end
-    end
-
-    def logger_appenders
-      appenders = [Logging.appenders.stdout(layout: stdout_layout)]
-
-      if log_file
-        FileUtils.mkdir_p(File.dirname(log_file))
-        appenders << Logging.appenders.file(log_file, layout: json_layout)
-      end
-
-      appenders
-    end
-
-    def log_file
-      config.logger.file
-    end
-
-    def json_layout
-      Logging.layouts.json(date_pattern: Constants::LOG_DATE_PATTERN)
-    end
-
-    def stdout_layout
-      if config.logger.stdout_json == true
-        json_layout
-      else
-        Logging.layouts.pattern(date_pattern: Constants::LOG_DATE_PATTERN)
-      end
-    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre.beta1
+  version: 2.1.1
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -12,10 +12,10 @@ authors:
 - Francisco Juan
 - Tommy Gustafsson
 - Daniel Orner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-04 00:00:00.000000000 Z
+date: 2022-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -299,7 +299,7 @@ licenses:
 - Apache License Version 2.0
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -310,12 +310,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Simplifying Kafka for ruby apps
 test_files: []