karafka 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1fcea5db98bb786e35298cda7dcfc34354935106
-  data.tar.gz: 8da4bad16ba1d23a04b6bfc6906d8432ff3d715c
+SHA256:
+  metadata.gz: 0bb0a1f72768ebf4bf720ebda57ebc26a0178275adabffd494b24e1612e9b38a
+  data.tar.gz: f586038a0498227e8a287cc173a23e77bb99b6cb8d453c39be55a220e2a0d361
 SHA512:
-  metadata.gz: c775dc7525a5e3f0252b5d5fb90f37d31636d2aeec99de936650ff6de6ec45c93383b111bd600d2cc190808e3b5f941dd619d8c5d51ad95c53852217df38184f
-  data.tar.gz: c891f17541b7298ac7e9e310d3d16a1e3464d365db34fcfb461109ddb6d03e3f499ccc1bd1886539d62d2451754adf4a7e9a4763324b5d46bcda680ca60ed358
+  metadata.gz: e2b862da6372bc91f76bc01c20eb21ccde9d61d67749ccce97eadd2f739484dbd68a149d08d6e275d94d95b6ce9610d86c2d01e85bb1841a80e048ed832810ac
+  data.tar.gz: 06fb89700e59f810ec984b54f0424cfdeb98c79e17b8629010f011e68edff5fa1e345576a7b28ed07aff440739ed11824016da6083601a92b8f83fcca68e576e

data/.ruby-version

@@ -1 +1 @@
-2.4.2
+2.5.0

data/.travis.yml

@@ -9,6 +9,7 @@ rvm:
   - 2.4.0
   - 2.4.1
   - 2.4.2
+  - 2.5.0
   - jruby-head
 script: bundle exec rspec spec/
 env:

data/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Karafka framework changelog
 
+## 1.2.0
+- Spec improvements
+- #260 - Specs missing randomization
+- #251 - Shutdown upon non responding (unreachable) cluster is not possible
+- #258 - Investigate lowering requirements on activesupport
+- #246 - Alias consumer#mark_as_consumed on controller
+- #259 - Allow forcing key/partition key on responders
+- #267 - Styling inconsistency
+- #242 - Support setting the max bytes to fetch per request
+- #247 - Support SCRAM once released
+- #271 - Provide an after_init option to pass a configuration block
+- #262 - Error in the monitor code for NewRelic
+- #241 - Performance metrics
+- #274 - Rename controllers to consumers
+- #184 - Seek to
+- #284 - Dynamic Params parent class
+- #275 - ssl_ca_certs_from_system
+- #296 - Instrument forceful exit with an error
+- Replaced some of the activesupport parts with dry-inflector
+- Lower ActiveSupport dependency
+- Remove configurators in favor of the after_init block configurator
+- Ruby 2.5.0 support
+- Renamed Karafka::Connection::Processor to Karafka::Connection::Delegator to match incoming naming conventions
+- Renamed Karafka::Connection::Consumer to Karafka::Connection::Client due to #274
+- Removed HashWithIndifferentAccess in favor of a regular hash
+- JSON parsing defaults now to string keys
+- Lower memory usage due to less params data internal details
+- Support multiple ```after_init``` blocks in favor of a single one
+- Renamed ```received_at``` to ```receive_time``` to follow ruby-kafka and WaterDrop conventions
+- Adjust internal setup to easier map Ruby-Kafka config changes
+- System callbacks reorganization
+- Added ```before_fetch_loop``` configuration block for early client usage (```#seek```, etc)
+- Renamed ```after_fetched``` to ```after_fetch``` to normalize the naming convention
+- Instrumentation on a connection delegator level
+- Added ```params_batch#last``` method to retrieve last element after unparsing
+- All params keys are now strings
+
+## 1.1.2
+- #256 - Default kafka.seed_brokers configuration is created in invalid format
+
 ## 1.1.1
 - #253 - Allow providing a global per app parser in config settings
 

data/Gemfile

@@ -5,8 +5,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
-  gem 'waterdrop'
-  gem 'timecop'
   gem 'rspec'
   gem 'simplecov'
+  gem 'timecop'
 end

data/Gemfile.lock

@@ -1,44 +1,56 @@
 PATH
   remote: .
   specs:
-    karafka (1.1.1)
-      activesupport (>= 5.0)
+    karafka (1.2.0)
+      activesupport (>= 4.0)
       dry-configurable (~> 0.7)
+      dry-inflector (~> 0.1.1)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       envlogic (~> 1.0)
       multi_json (>= 1.12)
       rake (>= 11.3)
       require_all (>= 1.4)
-      ruby-kafka (>= 0.5)
+      ruby-kafka (>= 0.5.3)
       thor (~> 0.19)
-      waterdrop (>= 1.0.1)
+      waterdrop (~> 1.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.1.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     concurrent-ruby (1.0.5)
-    delivery_boy (0.2.2)
+    delivery_boy (0.2.4)
       king_konf (~> 0.1.8)
-      ruby-kafka (~> 0.4)
+      ruby-kafka (~> 0.5.1)
     diff-lcs (1.3)
-    docile (1.1.5)
+    docile (1.3.0)
     dry-configurable (0.7.0)
       concurrent-ruby (~> 1.0)
     dry-container (0.6.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.1)
+    dry-core (0.4.5)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.2.0)
+    dry-events (0.1.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+    dry-inflector (0.1.1)
     dry-logic (0.4.2)
       dry-container (~> 0.2, >= 0.2.6)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
+    dry-monitor (0.1.2)
+      dry-configurable (~> 0.5)
+      dry-equalizer (~> 0.2)
+      dry-events (~> 0.1)
+      rouge (~> 2.0, >= 2.2.1)
     dry-types (0.12.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1)
@@ -54,23 +66,24 @@ GEM
       dry-equalizer (~> 0.2)
       dry-logic (~> 0.4, >= 0.4.0)
       dry-types (~> 0.12.0)
-    envlogic (1.0.4)
-      activesupport
-    i18n (0.9.1)
+    envlogic (1.1.0)
+      dry-inflector (~> 0.1)
+    i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     inflecto (0.0.2)
     json (2.1.0)
-    king_konf (0.1.8)
-    minitest (5.10.3)
-    multi_json (1.12.2)
-    null-logger (0.1.4)
-    rake (12.3.0)
-    require_all (1.4.0)
+    king_konf (0.1.10)
+    minitest (5.11.3)
+    multi_json (1.13.1)
+    null-logger (0.1.5)
+    rake (12.3.1)
+    require_all (2.0.0)
+    rouge (2.2.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
       rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.0)
+    rspec-core (3.7.1)
       rspec-support (~> 3.7.0)
     rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -78,24 +91,24 @@ GEM
     rspec-mocks (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
-    ruby-kafka (0.5.0)
-    simplecov (0.15.1)
-      docile (~> 1.1.0)
+    rspec-support (3.7.1)
+    ruby-kafka (0.5.4)
+    simplecov (0.16.1)
+      docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     thor (0.20.0)
     thread_safe (0.3.6)
     timecop (0.9.1)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    waterdrop (1.0.1)
-      delivery_boy (>= 0.2.2)
+    waterdrop (1.2.0)
+      delivery_boy (~> 0.2)
       dry-configurable (~> 0.7)
+      dry-monitor (~> 0.1)
       dry-validation (~> 0.11)
       null-logger
-      ruby-kafka (>= 0.5)
 
 PLATFORMS
   ruby
@@ -105,7 +118,6 @@ DEPENDENCIES
   rspec
   simplecov
   timecop
-  waterdrop
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

data/README.md

@@ -4,20 +4,23 @@
 
 Framework used to simplify Apache Kafka based Ruby applications development.
 
-It allows programmers to use approach similar to standard HTTP conventions (```params``` and ```params_batch```) when working with asynchronous Kafka messages.
+Karafka allows you to capture everything that happens in your systems in large scale, providing you with a seamless and stable core for consuming and processing this data, without having to focus on things that are not your business domain.
 
 Karafka not only handles incoming messages but also provides tools for building complex data-flow applications that receive and send messages.
 
+**Warning**: Wiki and all the docs refer to the 1.2.0.beta4. Sorry for the inconvenience. We will release the stable 1.2.0 version soon.
+
 ## How does it work
 
 Karafka provides a higher-level abstraction that allows you to focus on your business logic development, instead of focusing on implementing lower level abstraction layers. It provides developers with a set of tools that are dedicated for building multi-topic applications similarly to how Rails applications are being built.
 
 ### Some things you might wonder about:
 
-- You can integrate Karafka with any Ruby based application.
+- You can integrate Karafka with **any** Ruby based application.
 - Karafka does **not** require Sidekiq or any other third party software (apart from Kafka itself).
-- Karafka works with Ruby on Rails but it is a standalone framework that can work without it.
-- Karafka has a minimal set of dependencies, so adding it won't be a huge burden for your already existing applications.
+- Karafka works with Ruby on Rails but it is a **standalone** framework that can work without it.
+- Karafka has a **minimal** set of dependencies, so adding it won't be a huge burden for your already existing applications.
+- Karafka processes can be executed for a **given subset** of consumer groups and/or topics, so you can fine tune it depending on your business logic.
 
 Karafka based applications can be easily deployed to any type of infrastructure, including those based on:
 

data/karafka.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
@@ -16,16 +16,18 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'activesupport', '>= 5.0'
+  spec.add_dependency 'activesupport', '>= 4.0'
   spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'dry-inflector', '~> 0.1.1'
+  spec.add_dependency 'dry-monitor', '~> 0.1'
   spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'envlogic', '~> 1.0'
   spec.add_dependency 'multi_json', '>= 1.12'
   spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'require_all', '>= 1.4'
-  spec.add_dependency 'ruby-kafka', '>= 0.5'
+  spec.add_dependency 'ruby-kafka', '>= 0.5.3'
   spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'waterdrop', '>= 1.0.1'
+  spec.add_dependency 'waterdrop', '~> 1.2'
 
   spec.required_ruby_version = '>= 2.3.0'
 

data/lib/karafka.rb

@@ -11,10 +11,9 @@
   require_all
   dry-configurable
   dry-validation
+  dry/inflector
+  dry/monitor/notifications
   active_support/callbacks
-  active_support/core_ext/hash/indifferent_access
-  active_support/descendants_tracker
-  active_support/inflector
   karafka/loader
 ].each(&method(:require))
 
@@ -28,14 +27,14 @@ module Karafka
       @logger ||= App.config.logger
     end
 
-    # @return [::Karafka::Monitor] monitor that we want to use. Will use dummy monitor by default
+    # @return [::Karafka::Monitor] monitor that we want to use
     def monitor
       @monitor ||= App.config.monitor
     end
 
     # @return [String] root path of this gem
     def gem_root
-      Pathname.new(File.expand_path('../..', __FILE__))
+      Pathname.new(File.expand_path('..', __dir__))
     end
 
     # @return [String] Karafka app root path (user application path)
@@ -45,13 +44,13 @@ module Karafka
 
     # @return [String] path to Karafka gem root core
     def core_root
-      Pathname.new(File.expand_path('../karafka', __FILE__))
+      Pathname.new(File.expand_path('karafka', __dir__))
     end
 
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
-    #   It will be used for console/controllers/etc
+    #   It will be used for console/consumers/etc
     # @example Standard only-Karafka case
     #   Karafka.boot_file #=> '/home/app_path/karafka.rb'
     # @example Non standard case
@@ -63,6 +62,17 @@ module Karafka
   end
 end
 
+%w[
+  callbacks
+  callbacks/*
+  setup/dsl
+  setup/config
+  status
+  schemas/config
+  schemas/consumer_group_topic
+  schemas/consumer_group
+].each { |path| require_all File.join(Karafka.core_root, path + '.rb') }
+
 Karafka::Loader.load!(Karafka.core_root)
 Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
 Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/app.rb

@@ -3,14 +3,10 @@
 module Karafka
   # App class
   class App
-    class << self
-      # Sets up the whole configuration
-      # @param [Block] block configuration block
-      def setup(&block)
-        Setup::Config.setup(&block)
-        initialize!
-      end
+    extend Setup::Dsl
+    extend Callbacks::Dsl
 
+    class << self
       # Sets up all the internal components and bootstrap whole app
       # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
@@ -19,11 +15,7 @@ module Karafka
       def boot!
         Setup::Config.validate!
         Setup::Config.setup_components
-      end
-
-      # @return [Karafka::Config] config instance
-      def config
-        Setup::Config.config
+        Callbacks.after_init(Karafka::App.config)
       end
 
       # @return [Karafka::Routing::Builder] consumers builder instance
@@ -33,7 +25,7 @@ module Karafka
 
       Status.instance_methods(false).each do |delegated|
         define_method(delegated) do
-          Status.instance.public_send(delegated)
+          Status.instance.send(delegated)
         end
       end
 
@@ -41,10 +33,11 @@ module Karafka
       %i[
         root
         env
-        logger monitor
+        logger
+        monitor
       ].each do |delegated|
         define_method(delegated) do
-          Karafka.public_send(delegated)
+          Karafka.send(delegated)
         end
       end
     end

data/lib/karafka/attributes_map.rb

@@ -21,7 +21,7 @@ module Karafka
             offset_retention_time heartbeat_interval
           ],
           subscription: %i[start_from_beginning max_bytes_per_partition],
-          consuming: %i[min_bytes max_wait_time],
+          consuming: %i[min_bytes max_bytes max_wait_time],
           pausing: %i[pause_timeout],
           # All the options that are under kafka config namespace, but are not used
           # directly with kafka api, but from the Karafka user perspective, they are

data/lib/karafka/backends/inline.rb

@@ -9,8 +9,7 @@ module Karafka
 
       # Executes consume code immediately (without enqueuing)
       def process
-        Karafka.monitor.notice(self.class, params_batch)
-        consume
+        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
       end
     end
   end

data/lib/karafka/{base_controller.rb → base_consumer.rb}

@@ -2,24 +2,32 @@
 
 # Karafka module namespace
 module Karafka
-  # Base controller from which all Karafka controllers should inherit
-  class BaseController
+  # Base consumer from which all Karafka consumers should inherit
+  class BaseConsumer
     extend ActiveSupport::DescendantsTracker
+    extend Forwardable
+
+    # Allows us to mark messages as consumed for non-automatic mode without having
+    # to use consumer client directly. We do this that way, because most of the people should not
+    # mess with the client instance directly (just in case)
+    def_delegator :client, :mark_as_consumed
+
+    private :mark_as_consumed
 
     class << self
       attr_reader :topic
 
-      # Assigns a topic to a controller and build up proper controller functionalities, so it can
-      #   cooperate with the topic settings
+      # Assigns a topic to a consumer and builds up proper consumer functionalities
+      #   so that it can cooperate with the topic settings
       # @param topic [Karafka::Routing::Topic]
       # @return [Karafka::Routing::Topic] assigned topic
       def topic=(topic)
         @topic = topic
-        Controllers::Includer.call(self)
+        Consumers::Includer.call(self)
       end
     end
 
-    # @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
+    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
     def topic
       self.class.topic
     end
@@ -33,20 +41,20 @@ module Karafka
       @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
     end
 
-    # Executes the default controller flow.
+    # Executes the default consumer flow.
     def call
       process
     end
 
     private
 
-    # We make it private as it should be accesible only from the inside of a controller
+    # We make it private as it should be accessible only from the inside of a consumer
     attr_reader :params_batch
 
-    # @return [Karafka::Connection::Consumer] messages consumer that can be used to
+    # @return [Karafka::Connection::Client] messages consuming client that can be used to
     #    commit manually offset or pause / stop consumer based on the business logic
-    def consumer
-      Persistence::Consumer.read
+    def client
+      Persistence::Client.read
     end
 
     # Method that will perform business logic and on data received from Kafka (it will consume