karafka 1.2.2 → 1.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ba856816cda2f5e766137d0e61b9a2a4c554a462cbcf4b51816e8eb5e5b20a5
-  data.tar.gz: a12c0dc26c94916f4da4ceb7f195e429ab70011379b1475cc05e3fbb11a7de7d
+  metadata.gz: dbcc16b7a1dadd11283c52b9e9edd838a0f793e23aebf990aba78baf24c6efa1
+  data.tar.gz: a4da64f172baf0700faae318648b3007ef8406ce861122aa13104daae06213e6
 SHA512:
-  metadata.gz: 8d861156abd5026ef14ca5e029dda5f3c58829b6929852076c9da528f42c9e6c4ad81d3309cf7f2d4f08e421e7fe6f5f3d0a3b5ea3c5903e2ce9a773b5ebda47
-  data.tar.gz: 0257b70be38e2cfc9841dead5fb475a38523eb5064f6930ff062c84bdae746354ab37cb29028c9275089fd935f62c40fd5a8d1573a0c25ab63625aac922576d1
+  metadata.gz: b23b92b6c084d934cf64425f0d6a88ec63300972dd1f6c047f8e99be5e7f6233d2ca89af37d562fd9acee2025f60f8fdf4159bff1222ccb624c5b61526696922
+  data.tar.gz: d0ff78e7e938070b95d37b38cde37729a28cf26cc1bb47706fe848a8fe18a5a58f078b1f384e3ecefdcc07861cc4a18e66a68e8c674116f5e773acf2ea8988bb

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Karafka framework changelog
 
+## 1.2.3
+- #313 - support PLAINTEXT and SSL for scheme
+- #320 - Pausing indefinetely with nil pause timeout doesn't work
+- #318 - Partition pausing doesn't work with custom topic mappers
+- Rename ConfigAdapter to ApiAdapter to better reflect what it does
+- #317 - Manual offset committing doesn't work with custom topic mappers
+
 ## 1.2.2
 - #312 - Broken for ActiveSupport 5.2.0
 

data/Gemfile

@@ -5,6 +5,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
+  gem 'benchmark-ips'
   gem 'rspec'
   gem 'simplecov'
   gem 'timecop'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (1.2.2)
+    karafka (1.2.3)
       activesupport (>= 4.0)
       dry-configurable (~> 0.7)
       dry-inflector (~> 0.1.1)
@@ -23,10 +23,11 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    benchmark-ips (2.7.2)
     concurrent-ruby (1.0.5)
-    delivery_boy (0.2.4)
-      king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.5.1)
+    delivery_boy (0.2.6)
+      king_konf (~> 0.2)
+      ruby-kafka (~> 0.5)
     diff-lcs (1.3)
     docile (1.3.0)
     dry-configurable (0.7.0)
@@ -36,12 +37,12 @@ GEM
       dry-configurable (~> 0.1, >= 0.1.3)
     dry-core (0.4.5)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.0)
+    dry-equalizer (0.2.1)
     dry-events (0.1.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-    dry-inflector (0.1.1)
+    dry-inflector (0.1.2)
     dry-logic (0.4.2)
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
@@ -68,11 +69,11 @@ GEM
       dry-types (~> 0.12.0)
     envlogic (1.1.0)
       dry-inflector (~> 0.1)
-    i18n (1.0.0)
+    i18n (1.0.1)
       concurrent-ruby (~> 1.0)
     inflecto (0.0.2)
     json (2.1.0)
-    king_konf (0.1.10)
+    king_konf (0.2.0)
     minitest (5.11.3)
     multi_json (1.13.1)
     null-logger (0.1.5)
@@ -92,7 +93,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    ruby-kafka (0.5.4)
+    ruby-kafka (0.6.2)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -114,6 +115,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  benchmark-ips
   karafka!
   rspec
   simplecov

data/lib/karafka/attributes_map.rb

@@ -14,15 +14,15 @@ module Karafka
       # @note All other settings will be passed to Kafka.new method invocation.
       #   All elements in this hash are just edge cases
       # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      def config_adapter
+      def api_adapter
         {
           consumer: %i[
             session_timeout offset_commit_interval offset_commit_threshold
             offset_retention_time heartbeat_interval
           ],
-          subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_bytes max_wait_time],
-          pausing: %i[pause_timeout],
+          subscribe: %i[start_from_beginning max_bytes_per_partition],
+          consumption: %i[min_bytes max_bytes max_wait_time],
+          pause: %i[pause_timeout],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are
           # still related to kafka. They should not be proxied anywhere
@@ -32,7 +32,7 @@ module Karafka
 
       # @return [Array<Symbol>] properties that can be set on a per topic level
       def topic
-        (config_adapter[:subscription] + %i[
+        (api_adapter[:subscribe] + %i[
           backend
           name
           parser
@@ -48,10 +48,10 @@ module Karafka
       #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
       #   to add it to our configuration class and it will be handled automatically.
       def consumer_group
-        # @note We don't ignore the config_adapter[:ignored] values as they should be ignored
+        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
         #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = config_adapter[:subscription]
-        defined_settings = config_adapter.values.flatten
+        ignored_settings = api_adapter[:subscribe]
+        defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
         # This is a drity and bad hack of dry-configurable to get keys before setting values
         dynamically_proxied = Karafka::Setup::Config

data/lib/karafka/connection/{config_adapter.rb → api_adapter.rb}

@@ -3,22 +3,22 @@
 module Karafka
   # Namespace for all the things related to Kafka connection
   module Connection
-    # Mapper used to convert our internal settings into ruby-kafka settings
+    # Mapper used to convert our internal settings into ruby-kafka settings based on their
+    # API requirements.
     # Since ruby-kafka has more and more options and there are few "levels" on which
     # we have to apply them (despite the fact, that in Karafka you configure all of it
     # in one place), we have to remap it into what ruby-kafka driver requires
     # @note The good thing about Kafka.new method is that it ignores all options that
     #   do nothing. So we don't have to worry about injecting our internal settings
     #   into the client and breaking stuff
-    module ConfigAdapter
+    module ApiAdapter
       class << self
         # Builds all the configuration settings for Kafka.new method
-        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
         # @return [Array<Hash>] Array with all the client arguments including hash with all
         #   the settings required by Kafka.new method
         # @note We return array, so we can inject any arguments we want, in case of changes in the
         #   raw driver
-        def client(_consumer_group)
+        def client
           # This one is a default that takes all the settings except special
           # cases defined in the map
           settings = {
@@ -29,9 +29,9 @@ module Karafka
           kafka_configs.each do |setting_name, setting_value|
             # All options for config adapter should be ignored as we're just interested
             # in what is left, as we want to pass all the options that are "typical"
-            # and not listed in the config_adapter special cases mapping. All the values
-            # from the config_adapter mapping go somewhere else, not to the client directly
-            next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
+            # and not listed in the api_adapter special cases mapping. All the values
+            # from the api_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
 
             settings[setting_name] = setting_value
           end
@@ -58,26 +58,54 @@ module Karafka
         # @return [Array<Hash>] Array with all the arguments required by consuming method
         #   including hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
-        def consuming(consumer_group)
-          settings = {
-            automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
-          }
-          [sanitize(fetch_for(:consuming, consumer_group, settings))]
+        def consumption(consumer_group)
+          [
+            sanitize(
+              fetch_for(
+                :consumption,
+                consumer_group,
+                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+              )
+            )
+          ]
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
         # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
         # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
-        def subscription(topic)
-          settings = fetch_for(:subscription, topic)
+        def subscribe(topic)
+          settings = fetch_for(:subscribe, topic)
           [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
         end
 
         # Builds all the configuration settings required by kafka consumer#pause method
+        # @param topic [String] topic that we want to pause
+        # @param partition [Integer] number partition that we want to pause
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required to pause kafka consumer
-        def pausing(consumer_group)
-          { timeout: consumer_group.pause_timeout }
+        # @return [Array] array with all the details required to pause kafka consumer
+        def pause(topic, partition, consumer_group)
+          [
+            Karafka::App.config.topic_mapper.outgoing(topic),
+            partition,
+            { timeout: consumer_group.pause_timeout }
+          ]
+        end
+
+        # Remaps topic details taking the topic mapper feature into consideration.
+        # @param params [Karafka::Params::Params] params instance
+        # @return [Array] array with all the details needed by ruby-kafka to mark message
+        #   as processed
+        # @note When default empty topic mapper is used, no need for any conversion as the
+        #   internal and external format are exactly the same
+        def mark_message_as_processed(params)
+          # Majority of non heroku users don't use custom topic mappers. No need to change
+          # anything when it is a default mapper that does not change anything
+          return [params] if Karafka::App.config.topic_mapper == Karafka::Routing::TopicMapper
+
+          # @note We don't use tap as it is around 13% slower than non-dup version
+          dupped = params.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
+          [dupped]
         end
 
         private
@@ -90,7 +118,7 @@ module Karafka
         def fetch_for(namespace_key, route_layer, preexisting_settings = {})
           kafka_configs.each_key do |setting_name|
             # Ignore settings that are not related to our namespace
-            next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
             # Ignore settings that are already initialized
             # In case they are in preexisting settings fetched differently
             next if preexisting_settings.keys.include?(setting_name)

data/lib/karafka/connection/builder.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Builder used to construct Kafka client
+    module Builder
+      class << self
+        # Builds a Kafka::Cient instance that we use to work with Kafka cluster
+        # @return [::Kafka::Client] returns a Kafka client
+        def call
+          Kafka.new(*ApiAdapter.client)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/client.rb

@@ -23,7 +23,7 @@ module Karafka
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
       # @note This will yield with raw messages - no preprocessing or reformatting.
       def fetch_loop
-        settings = ConfigAdapter.consuming(consumer_group)
+        settings = ApiAdapter.consumption(consumer_group)
 
         if consumer_group.batch_fetching
           kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
@@ -66,10 +66,7 @@ module Karafka
       # @param topic [String] topic that we want to pause
       # @param partition [Integer] number partition that we want to pause
       def pause(topic, partition)
-        settings = ConfigAdapter.pausing(consumer_group)
-        timeout = settings[:timeout]
-        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
-        kafka_consumer.pause(topic, partition, settings)
+        kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
       end
 
       # Marks a given message as consumed and commit the offsets
@@ -77,7 +74,9 @@ module Karafka
       #   that offset commit happen asap in case of a crash
       # @param [Karafka::Params::Params] params message that we want to mark as processed
       def mark_as_consumed(params)
-        kafka_consumer.mark_message_as_processed(params)
+        kafka_consumer.mark_message_as_processed(
+          *ApiAdapter.mark_message_as_processed(params)
+        )
         # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
         # before the automatic triggers have kicked in.
         kafka_consumer.commit_offsets
@@ -90,11 +89,13 @@ module Karafka
       # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
       #   that is set up to consume from topics of a given consumer group
       def kafka_consumer
-        @kafka_consumer ||= kafka.consumer(
-          *ConfigAdapter.consumer(consumer_group)
+        # @note We don't cache the connection internally because we cache kafka_consumer that uses
+        #   kafka client object instance
+        @kafka_consumer ||= Builder.call.consumer(
+          *ApiAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|
-            consumer.subscribe(*ConfigAdapter.subscription(topic))
+            consumer.subscribe(*ApiAdapter.subscribe(topic))
           end
         end
       rescue Kafka::ConnectionError
@@ -105,13 +106,6 @@ module Karafka
         # down the road
         raise
       end
-
-      # @return [Kafka] returns a Kafka
-      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
-      #   object instance
-      def kafka
-        Kafka.new(*ConfigAdapter.client(consumer_group))
-      end
     end
   end
 end

data/lib/karafka/errors.rb

@@ -41,9 +41,6 @@ module Karafka
     # This should never happen and if it does, please contact us
     MissingClient = Class.new(BaseError)
 
-    # Raised when we attemp to pause a partition but the pause timeout is equal to 0
-    InvalidPauseTimeout = Class.new(BaseError)
-
     # Raised when want to hook up to an event that is not registered and supported
     UnregisteredMonitorEvent = Class.new(BaseError)
 

data/lib/karafka/process.rb

@@ -34,10 +34,8 @@ module Karafka
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
     # @note If there are no callbacks, this method will just ignore a given signal that was sent
-    # @yield [Block] block of code that we want to execute and supervise
     def supervise
       HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
-      yield
     end
 
     private

data/lib/karafka/schemas/consumer_group.rb

@@ -7,7 +7,7 @@ module Karafka
       # Valid uri schemas of Kafka broker url
       # The ||= is due to the behavior of require_all that resolves dependencies
       # but someetimes loads things twice
-      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
+      URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
 
       # Available sasl scram mechanism of authentication (plus nil)
       SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
@@ -31,7 +31,7 @@ module Karafka
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:seed_brokers).filled { each(:broker_schema?) }
       required(:session_timeout).filled { int? | float? }
-      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
+      required(:pause_timeout) { none? | ((int? | float?) & gteq?(0)) }
       required(:offset_commit_interval) { int? | float? }
       required(:offset_commit_threshold).filled(:int?)
       required(:offset_retention_time) { none?.not > int? }

data/lib/karafka/server.rb

@@ -19,9 +19,9 @@ module Karafka
 
       # Method which runs app
       def run
-        bind_on_sigint
-        bind_on_sigquit
-        bind_on_sigterm
+        process.on_sigint { stop_supervised }
+        process.on_sigquit { stop_supervised }
+        process.on_sigterm { stop_supervised }
         start_supervised
       end
 
@@ -39,29 +39,13 @@ module Karafka
         Karafka::Process.instance
       end
 
-      # What should happen when we decide to quit with sigint
-      def bind_on_sigint
-        process.on_sigint { stop_supervised }
-      end
-
-      # What should happen when we decide to quit with sigquit
-      def bind_on_sigquit
-        process.on_sigquit { stop_supervised }
-      end
-
-      # What should happen when we decide to quit with sigterm
-      def bind_on_sigterm
-        process.on_sigterm { stop_supervised }
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitily want to stop)
       def start_supervised
-        process.supervise do
-          Karafka::App.run!
-          Karafka::Fetcher.call
-        end
+        process.supervise
+        Karafka::App.run!
+        Karafka::Fetcher.call
       end
 
       # Stops Karafka with a supervision (as long as there is a shutdown timeout)

data/lib/karafka/setup/config.rb

@@ -73,7 +73,7 @@ module Karafka
         # Time that a given partition will be paused from fetching messages, when message
         # consumption fails. It allows us to process other partitions, while the error is being
         # resolved and also "slows" things down, so it prevents from "eating" up all messages and
-        # consuming them with failed code
+        # consuming them with failed code. Use `nil` if you want to pause forever and never retry.
         setting :pause_timeout, 10
         # option offset_commit_interval [Integer] the interval between offset commits,
         #   in seconds.

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.2.2'
+  VERSION = '1.2.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.2.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-10 00:00:00.000000000 Z
+date: 2018-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -222,8 +222,9 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/connection/api_adapter.rb
+- lib/karafka/connection/builder.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/config_adapter.rb
 - lib/karafka/connection/delegator.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/consumers/callbacks.rb
@@ -294,7 +295,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.4
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka