karafka 1.1.2 → 1.2.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c472e65702014af8a34972c3be5f9c552c71fbf7
-  data.tar.gz: e5664770c53adc058e6b41d0e6473022ac45276b
+SHA256:
+  metadata.gz: 3155a5ed8f8a5ebabcef64de6c30256d6555dfa4af20bbe48527a7080226fe35
+  data.tar.gz: 26c9e9adc3a23ce31848280d1d855069bd384e91f35e29e2ac6672c838082d01
 SHA512:
-  metadata.gz: bf3a7b57f04795b04987be56c8d1d805237efe32a6547f45bb32a15ad6184b62f8a920be96d67d45f431689f36b4c3d83ae3c52d426c5c99508da3cf7aa57672
-  data.tar.gz: e97ba211015095db1516c88e572c319013094ef90e22b6a0c1388ef3262c0417cefa170fa53fb669250bae59e3c6aea9358eb409cf640b8b7255798596c0f80e
+  metadata.gz: d9b791b522c88b9c594282679d9788d4fc76e7660525946c51e60d6c67857777bfb5d6b1fec60a5a9a695d1e5bee5ac8e19669d86f833f31b51f2055cb51dc16
+  data.tar.gz: a8aede80af84e97d26222908cb5b759e3c3e4d7da1bee802806511bdc9dcf013d03d902e8a77acdfcb38bbb46d7ba5dee5dffd2442a757b9781e4d41b2623580

data/.ruby-version

@@ -1 +1 @@
-2.4.2
+2.5.0

data/.travis.yml

@@ -9,6 +9,7 @@ rvm:
   - 2.4.0
   - 2.4.1
   - 2.4.2
+  - 2.5.0
   - jruby-head
 script: bundle exec rspec spec/
 env:

data/CHANGELOG.md

@@ -1,5 +1,39 @@
 # Karafka framework changelog
 
+## 1.2.0-beta1
+- Spec improvements
+- #260 - Specs missing randomization
+- #251 - Shutdown upon non responding (unreachable) cluster is not possible
+- #258 - Investigate lowering requirements on activesupport
+- #246 - Alias consumer#mark_as_consumed on controller
+- #259 - Allow forcing key/partition key on responders
+- #267 - Styling inconsistency
+- #242 - Support setting the max bytes to fetch per request
+- #247 - Support SCRAM once released
+- #271 - Provide an after_init option to pass a configuration block
+- #262 - Error in the monitor code for NewRelic
+- #241 - Performance metrics
+- #274 - Rename controllers to consumers
+- #184 - Seek to
+- #284 - Dynamic Params parent class
+- #275 - ssl_ca_certs_from_system
+- Replaced some of the activesupport parts with dry-inflector
+- Lower ActiveSupport dependency
+- Remove configurators in favor of the after_init block configurator
+- Ruby 2.5.0 support
+- Renamed Karafka::Connection::Processor to Karafka::Connection::Delegator to match incoming naming conventions
+- Renamed Karafka::Connection::Consumer to Karafka::Connection::Client due to #274
+- Removed HashWithIndifferentAccess in favor of a regular hash
+- JSON parsing defaults now to string keys
+- Lower memory usage due to less params data internal details
+- Support multiple ```after_init``` blocks in favor of a single one
+- Renamed ```received_at``` to ```receive_time``` to follow ruby-kafka and WaterDrop conventions
+- Adjust internal setup to easier map Ruby-Kafka config changes
+- System callbacks reorganization
+- Added ```before_fetch_loop``` configuration block for early client usage (```#seek```, etc)
+- Renamed ```after_fetched``` to ```after_fetch``` to normalize the naming convention
+- Instrumentation on a connection delegator level
+
 ## 1.1.2
 - #256 - Default kafka.seed_brokers configuration is created in invalid format
 

data/Gemfile

@@ -5,8 +5,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
-  gem 'waterdrop'
-  gem 'timecop'
   gem 'rspec'
   gem 'simplecov'
+  gem 'timecop'
 end

data/Gemfile.lock

@@ -1,30 +1,32 @@
 PATH
   remote: .
   specs:
-    karafka (1.1.2)
-      activesupport (>= 5.0)
+    karafka (1.2.0.beta1)
+      activesupport (>= 4.0)
       dry-configurable (~> 0.7)
+      dry-inflector (~> 0.1.1)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       envlogic (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
       require_all (>= 1.4)
-      ruby-kafka (>= 0.5)
+      ruby-kafka (>= 0.5.3)
       thor (~> 0.19)
-      waterdrop (>= 1.0.1)
+      waterdrop (>= 1.2.0.beta1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.1.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     concurrent-ruby (1.0.5)
-    delivery_boy (0.2.2)
+    delivery_boy (0.2.4)
       king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.4)
+      ruby-kafka (~> 0.5.1)
     diff-lcs (1.3)
     docile (1.1.5)
     dry-configurable (0.7.0)
@@ -32,13 +34,23 @@ GEM
     dry-container (0.6.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.1)
+    dry-core (0.4.4)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.2.0)
+    dry-events (0.1.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+    dry-inflector (0.1.1)
     dry-logic (0.4.2)
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
+    dry-monitor (0.1.2)
+      dry-configurable (~> 0.5)
+      dry-equalizer (~> 0.2)
+      dry-events (~> 0.1)
+      rouge (~> 2.0, >= 2.2.1)
     dry-types (0.12.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1)
@@ -56,21 +68,22 @@ GEM
       dry-types (~> 0.12.0)
     envlogic (1.0.4)
       activesupport
-    i18n (0.9.1)
+    i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     inflecto (0.0.2)
     json (2.1.0)
-    king_konf (0.1.8)
-    minitest (5.10.3)
-    multi_json (1.12.2)
+    king_konf (0.1.10)
+    minitest (5.11.3)
+    multi_json (1.13.1)
     null-logger (0.1.4)
     rake (12.3.0)
-    require_all (1.4.0)
+    require_all (1.5.0)
+    rouge (2.2.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
       rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.0)
+    rspec-core (3.7.1)
       rspec-support (~> 3.7.0)
     rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -78,8 +91,8 @@ GEM
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
-    ruby-kafka (0.5.0)
+    rspec-support (3.7.1)
+    ruby-kafka (0.5.3)
     simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
@@ -88,14 +101,15 @@ GEM
     thor (0.20.0)
     thread_safe (0.3.6)
     timecop (0.9.1)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.0.1)
-      delivery_boy (>= 0.2.2)
+    waterdrop (1.2.0.beta1)
+      delivery_boy (>= 0.2.3)
       dry-configurable (~> 0.7)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       null-logger
-      ruby-kafka (>= 0.5)
+      ruby-kafka (>= 0.5.3)
 
 PLATFORMS
   ruby
@@ -105,7 +119,6 @@ DEPENDENCIES
   rspec
   simplecov
   timecop
-  waterdrop
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

data/README.md

@@ -14,10 +14,11 @@ Karafka provides a higher-level abstraction that allows you to focus on your bus
 
 ### Some things you might wonder about:
 
-- You can integrate Karafka with any Ruby based application.
+- You can integrate Karafka with **any** Ruby based application.
 - Karafka does **not** require Sidekiq or any other third party software (apart from Kafka itself).
-- Karafka works with Ruby on Rails but it is a standalone framework that can work without it.
-- Karafka has a minimal set of dependencies, so adding it won't be a huge burden for your already existing applications.
+- Karafka works with Ruby on Rails but it is a **standalone** framework that can work without it.
+- Karafka has a **minimal** set of dependencies, so adding it won't be a huge burden for your already existing applications.
+- Karafka processes can be executed for a **given subset** of consumer groups and/or topics, so you can fine tune it depending on your business logic.
 
 Karafka based applications can be easily deployed to any type of infrastructure, including those based on:
 

data/karafka.gemspec

@@ -16,16 +16,18 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'activesupport', '>= 5.0'
+  spec.add_dependency 'activesupport', '>= 4.0'
   spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'dry-inflector', '~> 0.1.1'
+  spec.add_dependency 'dry-monitor', '~> 0.1'
   spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'envlogic', '~> 1.0'
   spec.add_dependency 'multi_json', '>= 1.12'
   spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'require_all', '>= 1.4'
-  spec.add_dependency 'ruby-kafka', '>= 0.5'
+  spec.add_dependency 'ruby-kafka', '>= 0.5.3'
   spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'waterdrop', '>= 1.0.1'
+  spec.add_dependency 'waterdrop', '>= 1.2.0.beta1'
 
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka.rb

@@ -11,10 +11,9 @@
   require_all
   dry-configurable
   dry-validation
+  dry/inflector
+  dry/monitor/notifications
   active_support/callbacks
-  active_support/core_ext/hash/indifferent_access
-  active_support/descendants_tracker
-  active_support/inflector
   karafka/loader
 ].each(&method(:require))
 
@@ -28,7 +27,7 @@ module Karafka
       @logger ||= App.config.logger
     end
 
-    # @return [::Karafka::Monitor] monitor that we want to use. Will use dummy monitor by default
+    # @return [::Karafka::Monitor] monitor that we want to use
     def monitor
       @monitor ||= App.config.monitor
     end
@@ -51,7 +50,7 @@ module Karafka
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
-    #   It will be used for console/controllers/etc
+    #   It will be used for console/consumers/etc
     # @example Standard only-Karafka case
     #   Karafka.boot_file #=> '/home/app_path/karafka.rb'
     # @example Non standard case

data/lib/karafka/app.rb

@@ -3,14 +3,10 @@
 module Karafka
   # App class
   class App
-    class << self
-      # Sets up the whole configuration
-      # @param [Block] block configuration block
-      def setup(&block)
-        Setup::Config.setup(&block)
-        initialize!
-      end
+    extend Setup::Dsl
+    extend Callbacks::Dsl
 
+    class << self
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
@@ -19,11 +15,7 @@ module Karafka
       def boot!
         Setup::Config.validate!
         Setup::Config.setup_components
-      end
-
-      # @return [Karafka::Config] config instance
-      def config
-        Setup::Config.config
+        Callbacks.after_init(Karafka::App.config)
       end
 
       # @return [Karafka::Routing::Builder] consumers builder instance
@@ -33,7 +25,7 @@ module Karafka
 
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
-          Status.instance.public_send(delegated)
+          Status.instance.send(delegated)
         end
       end
 
@@ -41,10 +33,11 @@ module Karafka
       %i[
         root
         env
-        logger monitor
+        logger
+        monitor
       ].each do |delegated|
         define_method(delegated) do
-          Karafka.public_send(delegated)
+          Karafka.send(delegated)
         end
       end
     end

data/lib/karafka/attributes_map.rb

@@ -21,7 +21,7 @@ module Karafka
             offset_retention_time heartbeat_interval
           ],
           subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_wait_time],
+          consuming: %i[min_bytes max_bytes max_wait_time],
           pausing: %i[pause_timeout],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are

data/lib/karafka/backends/inline.rb

@@ -9,8 +9,7 @@ module Karafka
 
       # Executes consume code immediately (without enqueuing)
       def process
-        Karafka.monitor.notice(self.class, params_batch)
-        consume
+        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
       end
     end
   end

data/lib/karafka/{base_controller.rb → base_consumer.rb}

@@ -2,24 +2,32 @@
 
 # Karafka module namespace
 module Karafka
-  # Base controller from which all Karafka controllers should inherit
-  class BaseController
+  # Base consumer from which all Karafka consumers should inherit
+  class BaseConsumer
     extend ActiveSupport::DescendantsTracker
+    extend Forwardable
+
+    # Allows us to mark messages as consumed for non-automatic mode without having
+    # to use consumer client directly. We do this that way, because most of the people should not
+    # mess with the client instance directly (just in case)
+    def_delegator :client, :mark_as_consumed
+
+    private :mark_as_consumed
 
     class << self
       attr_reader :topic
 
-      # Assigns a topic to a controller and build up proper controller functionalities, so it can
-      #   cooperate with the topic settings
+      # Assigns a topic to a consumer and builds up proper consumer functionalities
+      #   so that it can cooperate with the topic settings
       # @param topic [Karafka::Routing::Topic]
       # @return [Karafka::Routing::Topic] assigned topic
       def topic=(topic)
         @topic = topic
-        Controllers::Includer.call(self)
+        Consumers::Includer.call(self)
       end
     end
 
-    # @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
     def topic
       self.class.topic
     end
@@ -33,20 +41,20 @@ module Karafka
       @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
     end
 
-    # Executes the default controller flow.
+    # Executes the default consumer flow.
     def call
       process
     end
 
     private
 
-    # We make it private as it should be accesible only from the inside of a controller
+    # We make it private as it should be accessible only from the inside of a consumer
     attr_reader :params_batch
 
-    # @return [Karafka::Connection::Consumer] messages consumer that can be used to
+    # @return [Karafka::Connection::Client] messages consuming client that can be used to
     #    commit manually offset or pause / stop consumer based on the business logic
-    def consumer
-      Persistence::Consumer.read
+    def client
+      Persistence::Client.read
     end
 
     # Method that will perform business logic and on data received from Kafka (it will consume

data/lib/karafka/base_responder.rb

@@ -62,6 +62,11 @@ module Karafka
     # Definitions of all topics that we want to be able to use in this responder should go here
     class_attribute :topics
 
+    # Schema that we can use to control and/or require some additional details upon options
+    # that are being passed to the producer. This can be in particular useful if we want to make
+    # sure that for example partition_key is always present.
+    class_attribute :options_schema
+
     attr_reader :messages_buffer
 
     class << self
@@ -108,7 +113,8 @@ module Karafka
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
     def call(*data)
       respond(*data)
-      validate!
+      validate_usage!
+      validate_options!
       deliver!
     end
 
@@ -116,7 +122,7 @@ module Karafka
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
     # a registered required topic, etc.
-    def validate!
+    def validate_usage!
       registered_topics = self.class.topics.map do |name, topic|
         topic.to_h.merge!(
           usage_count: messages_buffer[name]&.count || 0
@@ -138,20 +144,26 @@ module Karafka
       raise Karafka::Errors::InvalidResponderUsage, result.errors
     end
 
+    # Checks if we met all the options requirements before sending them to the producer.
+    def validate_options!
+      return true unless self.class.options_schema
+
+      messages_buffer.each_value do |messages_set|
+        messages_set.each do |message_data|
+          result = self.class.options_schema.call(message_data.last)
+          next if result.success?
+          raise Karafka::Errors::InvalidResponderMessageOptions, result.errors
+        end
+      end
+    end
+
     # Takes all the messages from the buffer and delivers them one by one
     # @note This method is executed after the validation, so we're sure that
     #   what we send is legit and it will go to a proper topics
     def deliver!
-      messages_buffer.each do |topic, data_elements|
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
-
+      messages_buffer.each_value do |data_elements|
         data_elements.each do |data, options|
-          producer(options).call(
-            data,
-            options.merge(topic: mapped_topic)
-          )
+          producer(options).call(data, options)
         end
       end
     end
@@ -170,10 +182,17 @@ module Karafka
     # @param options [Hash] options for waterdrop (e.g. partition_key)
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
-      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
+      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
+      # string topics
+      topic = topic.to_s
 
-      messages_buffer[topic.to_s] ||= []
-      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
+      messages_buffer[topic] ||= []
+      messages_buffer[topic] << [
+        @parser_class.generate(data),
+        # We map this topic name, so it will match namespaced/etc topic in Kafka
+        # @note By default will not change topic (if default mapper used)
+        options.merge(topic: Karafka::App.config.topic_mapper.outgoing(topic))
+      ]
     end
 
     # @param options [Hash] options for waterdrop