phobos 1.9.0.pre.beta3 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37dfd76749d3cbea4d6e7611ad70581eb938487ff37088e6f80d2b5ba78eb844
-  data.tar.gz: ba3104759ff0d2d2369b5077abc487545f75b32815e4fb4af042575e4176c12f
+  metadata.gz: ec66514784d6b38c90a4ca0aaf063e001536d2680732c4afa535b49c2a55cb9c
+  data.tar.gz: 1d988dd76e39305cfb30d3dd8e8fd926065631e391ed077d7c0459e6760587b4
 SHA512:
-  metadata.gz: 2cfb3e98b26a94f2adfd3ab579bc47aae565ece64bbdec1446833e33cb70c0e27e4413d0d4d083c5dc2cdf04827624c64c3ad51781c686f83013dfe21b363926
-  data.tar.gz: 829f7fd8355cc7717535e9e6f108a6f3dd18dd25f144ae659b4c81db9a19e661a8c916caf8ab9891d49e8502c5027d68376ceb7fc2ea8632b3142efe88c8b414
+  metadata.gz: 07a92fdf61d970a61f98cfb8615568fa48af0550fb473458b6dce67b479b684ce8e280dbd97b20b5c16f84bf14fa18eea47f8bae490c6ac5712b8d21d2c4d037
+  data.tar.gz: 59bb2c58cfd017b8f9462eb431d2bb62322bdc817a1260816648481abff4ff0a5d6f1df5430f3888a0bad3cb92eeb11ee2ace3ced62df1e8ee02756bb1e8a795

data/.gitignore

@@ -12,3 +12,5 @@ spec/examples.txt
 config/*.yml
 log/*.log
 .byebug_history
+.idea
+*.gem

data/CHANGELOG.md

@@ -6,10 +6,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ``
 ## UNRELEASED
 
+## [2.1.0] - 2021-05-27
+
+- Modify config to allow specifying kafka connection information separately for consumers and producers
+
+## [2.0.2] - 2021-01-28
+- Additional fixes for Ruby 3.0
+
+## [2.0.1] - 2021-01-14
+- Fix bug with Ruby 3.0 around OpenStruct
+
+## [2.0.0-beta1] - 2020-05-04
+
+- Remove deprecated patterns:
+-- `before_consume` method
+-- `around_consume` as a class method or without yielding values
+-- `publish` and `async_publish` with positional arguments
+
+## [1.9.0] - 2020-03-05
+- Bumped version to 1.9.0.
+
 ## [1.9.0-beta3] - 2020-02-05
+
 - Fix bug where deprecation errors would be shown when receiving nil payloads
   even if `around_consume` was updated to yield them.
 
+
 ## [1.9.0-beta2] - 2020-01-09
 
 - Allow `around_consume` to yield payload and metadata, and deprecate

data/README.md

@@ -21,7 +21,6 @@ With Phobos by your side, all this becomes smooth sailing.
 ## Table of Contents
 
 1. [Installation](#installation)
-  1. [Upgrade Notes](#upgrade-notes)
 1. [Usage](#usage)
   1. [Standalone apps](#usage-standalone-apps)
   1. [Consuming messages from Kafka](#usage-consuming-messages-from-kafka)
@@ -32,6 +31,7 @@ With Phobos by your side, all this becomes smooth sailing.
 1. [Plugins](#plugins)
 1. [Development](#development)
 1. [Test](#test)
+1. [Upgrade Notes](#upgrade-notes)
 
 ## <a name="installation"></a> Installation
 
@@ -53,10 +53,6 @@ Or install it yourself as:
 $ gem install phobos
 ```
 
-### <a name="upgrade-notes"></a> Upgrade Notes
-
-Version 1.8.2 introduced a new `persistent_connections` setting for regular producers. This reduces the number of connections used to produce messages and you should consider setting it to true. This does require a manual shutdown call -  please see [Producers with persistent connections](#persistent-connection).
-
 ## <a name="usage"></a> Usage
 
 Phobos can be used in two ways: as a standalone application or to support Kafka features in your existing project - including Rails apps. It provides a CLI tool to run it.
@@ -197,8 +193,6 @@ class MyHandler
 end
 ```
 
-Note: Previous versions used a `before_consume` method to pre-process the payload. This is still supported, but deprecated. Going forward, `around_consume` should yield the payload and metadata back to the calling code, allowing it to act as a pre-processor, e.g. by decoding Avro messages into Ruby hashes.
-
 Take a look at the examples folder for some ideas.
 
 The hander life cycle can be illustrated as:
@@ -287,8 +281,6 @@ my = MyProducer.new
 my.producer.publish(topic: 'topic', payload: 'message-payload', key: 'partition and message key', headers: { header_1: 'value 1' })
 ```
 
-Older versions of Phobos provided a `publish` method that accepted positional arguments. That version is still supported but it's soon to be deprecated in favour of the keyword arguments version.
-
 It is also possible to publish several messages at once:
 
 ```ruby
@@ -412,9 +404,12 @@ All [options supported by  `ruby-kafka`][ruby-kafka-client] can be provided.
 __producer__ provides configurations for all producers created over the application,
 the options are the same for regular and async producers.
 All [options supported by  `ruby-kafka`][ruby-kafka-producer] can be provided.
+If the __kafka__ key is present under __producer__, it is merged into the top-level __kafka__, allowing different connection configuration for producers.
 
 __consumer__ provides configurations for all consumer groups created over the application.
 All [options supported by  `ruby-kafka`][ruby-kafka-consumer] can be provided.
+If the __kafka__ key is present under __consumer__, it is merged into the top-level __kafka__, allowing different connection configuration for consumers.
+
 
 __backoff__ Phobos provides automatic retries for your handlers. If an exception
 is raised, the listener will retry following the back off configured here.
@@ -590,8 +585,11 @@ After checking out the repo:
 * make sure `docker` is installed and running (for windows and mac this also includes `docker-compose`).
 * Linux: make sure `docker-compose` is installed and running.
 * run `bin/setup` to install dependencies
-* run `docker-compose up` to start the required kafka containers in a window
-* run `rspec` to run the tests in another window
+* run `docker-compose up -d --force-recreate kafka zookeeper` to start the required kafka containers
+* run tests to confirm no environmental issues
+  * wait a few seconds for kafka broker to get set up - `sleep 30`
+  * run `docker-compose run --rm test`
+  * make sure it reports `X examples, 0 failures`
 
 You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
@@ -604,8 +602,16 @@ Phobos exports a spec helper that can help you test your consumer. The Phobos li
 * `process_message(handler:, payload:, metadata: {}, encoding: nil)` - Invokes your handler with payload and metadata, using a dummy listener (encoding and metadata are optional).
 
 ```ruby
-require 'spec_helper'
+### spec_helper.rb
+require 'phobos/test/helper'
+RSpec.configure do |config|
+  config.include Phobos::Test::Helper
+  config.before(:each) do
+    Phobos.configure(path_to_my_config_file)
+  end
+end 
 
+### Spec file
 describe MyConsumer do
   let(:payload) { 'foo' }
   let(:metadata) { Hash(foo: 'bar') }
@@ -619,6 +625,57 @@ describe MyConsumer do
 end
 ```
 
+## <a name="upgrade-notes"></a> Upgrade Notes
+
+Version 2.0 removes deprecated ways of defining producers and consumers:
+* The `before_consume` method has been removed. You can have this behavior in the first part of an `around_consume` method.
+* `around_consume` is now only available as an instance method, and it must yield the values to pass to the `consume` method.
+* `publish` and `async_publish` now only accept keyword arguments, not positional arguments.
+
+Example pre-2.0:
+```ruby
+class MyHandler
+  include Phobos::Handler
+
+  def before_consume(payload, metadata)
+    payload[:id] = 1
+  end
+
+  def self.around_consume(payload, metadata)
+    metadata[:key] = 5
+    yield
+  end
+end
+```
+
+In 2.0:
+```ruby
+class MyHandler
+  include Phobos::Handler
+
+  def around_consume(payload, metadata)
+    new_payload = payload.dup
+    new_metadata = metadata.dup
+    new_payload[:id] = 1
+    new_metadata[:key] = 5
+    yield new_payload, new_metadata
+  end
+end
+```
+
+Producer, 1.9:
+```ruby
+  producer.publish('my-topic', { payload_value: 1}, 5, 3, {header_val: 5})
+```
+
+Producer 2.0:
+```ruby
+  producer.publish(topic: 'my-topic', payload: { payload_value: 1}, key: 5, 
+     partition_key: 3, headers: { header_val: 5})
+```
+
+Version 1.8.2 introduced a new `persistent_connections` setting for regular producers. This reduces the number of connections used to produce messages and you should consider setting it to true. This does require a manual shutdown call -  please see [Producers with persistent connections](#persistent-connection).
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/klarna/phobos.

data/config/phobos.yml.example

@@ -65,6 +65,10 @@ producer:
   # that you need to manually call sync_producer_shutdown before exiting,
   # similar to async_producer_shutdown.
   persistent_connections: false
+  # kafka here supports the same parameters as the top-level, allowing custom connection
+  # configuration details for producers
+  kafka:
+    connect_timeout: 120
 
 consumer:
   # number of seconds after which, if a client hasn't contacted the Kafka cluster,
@@ -79,6 +83,10 @@ consumer:
   offset_retention_time:
   # interval between heartbeats; must be less than the session window
   heartbeat_interval: 10
+  # kafka here supports the same parameters as the top-level, allowing custom connection
+  # configuration details for consumers
+  kafka:
+    connect_timeout: 130
 
 backoff:
   min_ms: 1000

data/lib/phobos.rb

@@ -55,8 +55,12 @@ module Phobos
 
     def configure(configuration)
       @config = fetch_configuration(configuration)
-      @config.class.send(:define_method, :producer_hash) { Phobos.config.producer&.to_hash }
-      @config.class.send(:define_method, :consumer_hash) { Phobos.config.consumer&.to_hash }
+      @config.class.send(:define_method, :producer_hash) do
+        Phobos.config.producer&.to_hash&.except(:kafka)
+      end
+      @config.class.send(:define_method, :consumer_hash) do
+        Phobos.config.consumer&.to_hash&.except(:kafka)
+      end
       @config.listeners ||= []
       configure_logger
     end
@@ -66,8 +70,14 @@ module Phobos
       @config.listeners += listeners_config.listeners
     end
 
-    def create_kafka_client
-      Kafka.new(config.kafka.to_hash.merge(logger: @ruby_kafka_logger))
+    def create_kafka_client(config_key = nil)
+      kafka_config = config.kafka.to_hash.merge(logger: @ruby_kafka_logger)
+
+      if config_key
+        kafka_config = kafka_config.merge(**config.send(config_key)&.kafka&.to_hash || {})
+      end
+
+      Kafka.new(**kafka_config)
     end
 
     def create_exponential_backoff(backoff_config = nil)

data/lib/phobos/actions/process_batch_inline.rb

@@ -47,32 +47,13 @@ module Phobos
         )
       end
 
-      def preprocess(batch, handler)
-        if handler.respond_to?(:before_consume_batch)
-          Phobos.deprecate('before_consume_batch is deprecated and will be removed in 2.0. \
-                            Use around_consume_batch and yield payloads and metadata objects.')
-          handler.before_consume_batch(batch, @metadata)
-        else
-          batch
-        end
-      end
-
       def process_batch(batch)
         instrument('listener.process_batch_inline', @metadata) do |_metadata|
           handler = @listener.handler_class.new
 
-          preprocessed_batch = preprocess(batch, handler)
-          consume_block = proc { |around_batch, around_metadata|
-            if around_batch
-              handler.consume_batch(around_batch, around_metadata)
-            else
-              Phobos.deprecate('Calling around_consume_batch without yielding payloads \
-                                and metadata is deprecated and will be removed in 2.0.')
-              handler.consume_batch(preprocessed_batch, @metadata)
-            end
-          }
-
-          handler.around_consume_batch(preprocessed_batch, @metadata, &consume_block)
+          handler.around_consume_batch(batch, @metadata) do |around_batch, around_metadata|
+            handler.consume_batch(around_batch, around_metadata)
+          end
         end
       end
     end

data/lib/phobos/actions/process_message.rb

@@ -35,47 +35,12 @@ module Phobos
 
       private
 
-      def preprocess(payload, handler)
-        if handler.respond_to?(:before_consume)
-          Phobos.deprecate('before_consume is deprecated and will be removed in 2.0. \
-                            Use around_consume and yield payload and metadata objects.')
-          begin
-            handler.before_consume(payload, @metadata)
-          rescue ArgumentError
-            handler.before_consume(payload)
-          end
-        else
-          payload
-        end
-      end
-
-      def consume_block(payload, handler)
-        proc { |around_payload, around_metadata|
-          if around_metadata
-            handler.consume(around_payload, around_metadata)
-          else
-            Phobos.deprecate('Calling around_consume without yielding payload and metadata \
-                              is deprecated and will be removed in 2.0.')
-            handler.consume(payload, @metadata)
-          end
-        }
-      end
-
       def process_message(payload)
         instrument('listener.process_message', @metadata) do
           handler = @listener.handler_class.new
 
-          preprocessed_payload = preprocess(payload, handler)
-          block = consume_block(preprocessed_payload, handler)
-
-          if @listener.handler_class.respond_to?(:around_consume)
-            # around_consume class method implementation
-            Phobos.deprecate('around_consume has been moved to instance method, please update '\
-                             'your consumer. This will not be backwards compatible in the future.')
-            @listener.handler_class.around_consume(preprocessed_payload, @metadata, &block)
-          else
-            # around_consume instance method implementation
-            handler.around_consume(preprocessed_payload, @metadata, &block)
+          handler.around_consume(payload, @metadata) do |around_payload, around_metadata|
+            handler.consume(around_payload, around_metadata)
           end
         end
       end

data/lib/phobos/deep_struct.rb

@@ -8,7 +8,7 @@ module Phobos
     # Based on
     # https://docs.omniref.com/ruby/2.3.0/files/lib/ostruct.rb#line=88
     def initialize(hash = nil)
-      @table = {}
+      super
       @hash_table = {}
 
       hash&.each_pair do |key, value|

data/lib/phobos/executor.rb

@@ -13,7 +13,7 @@ module Phobos
         max_concurrency = listener_configs[:max_concurrency] || 1
         Array.new(max_concurrency).map do
           configs = listener_configs.select { |k| Constants::LISTENER_OPTS.include?(k) }
-          Phobos::Listener.new(configs.merge(handler: handler_class))
+          Phobos::Listener.new(**configs.merge(handler: handler_class))
         end
       end
     end

data/lib/phobos/listener.rb

@@ -35,7 +35,7 @@ module Phobos
       )
       @encoding = Encoding.const_get(force_encoding.to_sym) if force_encoding
       @message_processing_opts = compact(min_bytes: min_bytes, max_wait_time: max_wait_time)
-      @kafka_client = Phobos.create_kafka_client
+      @kafka_client = Phobos.create_kafka_client(:consumer)
       @producer_enabled = @handler_class.ancestors.include?(Phobos::Producer)
     end
     # rubocop:enable Metrics/MethodLength
@@ -93,7 +93,7 @@ module Phobos
     def start_listener
       instrument('listener.start', listener_metadata) do
         @consumer = create_kafka_consumer
-        @consumer.subscribe(topic, @subscribe_opts)
+        @consumer.subscribe(topic, **@subscribe_opts)
 
         # This is done here because the producer client is bound to the current thread and
         # since "start" blocks a thread might be used to call it
@@ -135,7 +135,7 @@ module Phobos
     end
 
     def consume_each_batch
-      @consumer.each_batch(@message_processing_opts) do |batch|
+      @consumer.each_batch(**@message_processing_opts) do |batch|
         batch_processor = Phobos::Actions::ProcessBatch.new(
           listener: self,
           batch: batch,
@@ -149,7 +149,7 @@ module Phobos
     end
 
     def consume_each_batch_inline
-      @consumer.each_batch(@message_processing_opts) do |batch|
+      @consumer.each_batch(**@message_processing_opts) do |batch|
         batch_processor = Phobos::Actions::ProcessBatchInline.new(
           listener: self,
           batch: batch,
@@ -163,7 +163,7 @@ module Phobos
     end
 
     def consume_each_message
-      @consumer.each_message(@message_processing_opts) do |message|
+      @consumer.each_message(**@message_processing_opts) do |message|
         message_processor = Phobos::Actions::ProcessMessage.new(
           listener: self,
           message: message,
@@ -181,7 +181,7 @@ module Phobos
         Constants::KAFKA_CONSUMER_OPTS.include?(k)
       end
       configs.merge!(@kafka_consumer_opts)
-      @kafka_client.consumer({ group_id: group_id }.merge(configs))
+      @kafka_client.consumer(**{ group_id: group_id }.merge(configs))
     end
 
     def compact(hash)

data/lib/phobos/producer.rb

@@ -11,28 +11,24 @@ module Phobos
     end
 
     class PublicAPI
-      MissingRequiredArgumentsError = Class.new(StandardError) do
-        def initialize
-          super('You need to provide a topic name and a payload')
-        end
-      end
-
       def initialize(host_obj)
         @host_obj = host_obj
       end
 
-      def publish(*args, **kwargs)
-        Phobos.deprecate(deprecate_positional_args_message('publish')) if kwargs.empty?
-
-        args = normalize_arguments(*args, **kwargs)
-        class_producer.publish(**args)
+      def publish(topic:, payload:, key: nil, partition_key: nil, headers: nil)
+        class_producer.publish(topic: topic,
+                               payload: payload,
+                               key: key,
+                               partition_key: partition_key,
+                               headers: headers)
       end
 
-      def async_publish(*args, **kwargs)
-        Phobos.deprecate(deprecate_positional_args_message('async_publish')) if kwargs.empty?
-
-        args = normalize_arguments(*args, **kwargs)
-        class_producer.async_publish(**args)
+      def async_publish(topic:, payload:, key: nil, partition_key: nil, headers: nil)
+        class_producer.async_publish(topic: topic,
+                                     payload: payload,
+                                     key: key,
+                                     partition_key: partition_key,
+                                     headers: headers)
       end
 
       # @param messages [Array(Hash(:topic, :payload, :key, :headers))]
@@ -54,36 +50,6 @@ module Phobos
       def class_producer
         @host_obj.class.producer
       end
-
-      # rubocop:disable Metrics/ParameterLists
-      def normalize_arguments(p_topic = nil, p_payload = nil, p_key = nil,
-                              p_partition_key = nil, p_headers = {},
-                              **kwargs)
-        {}.tap do |args|
-          {
-            topic: p_topic,
-            payload: p_payload,
-            key: p_key,
-            partition_key: p_partition_key,
-            headers: p_headers
-          }.each { |k, v| args[k] = kwargs[k] || v }
-
-          raise MissingRequiredArgumentsError if [:topic, :payload].any? { |k| args[k].nil? }
-
-          kwargs.each do |k, v|
-            next if args.key?(k)
-
-            args[:headers][k] = v
-          end
-        end
-      end
-      # rubocop:enable Metrics/ParameterLists
-
-      def deprecate_positional_args_message(method_name)
-        "The `#{method_name}` method should now receive keyword arguments " \
-          'rather than positional ones. Please update your publishers. This will ' \
-          'not be backwards compatible in the future.'
-      end
     end
 
     module ClassMethods
@@ -111,8 +77,8 @@ module Phobos
         end
 
         def create_sync_producer
-          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
-          sync_producer = client.producer(regular_configs)
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client(:producer))
+          sync_producer = client.producer(**regular_configs)
           if Phobos.config.producer_hash[:persistent_connections]
             producer_store[:sync_producer] = sync_producer
           end
@@ -142,8 +108,8 @@ module Phobos
         end
 
         def create_async_producer
-          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
-          async_producer = client.async_producer(async_configs)
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client(:producer))
+          async_producer = client.async_producer(**async_configs)
           producer_store[:async_producer] = async_producer
         end
 

data/lib/phobos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phobos
-  VERSION = '1.9.0-beta3'
+  VERSION = '2.1.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 1.9.0.pre.beta3
+  version: 2.1.0
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -12,10 +12,10 @@ authors:
 - Francisco Juan
 - Tommy Gustafsson
 - Daniel Orner
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-05 00:00:00.000000000 Z
+date: 2021-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -299,7 +299,7 @@ licenses:
 - Apache License Version 2.0
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -310,12 +310,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.16
+signing_key:
 specification_version: 4
 summary: Simplifying Kafka for ruby apps
 test_files: []