karafka 1.1.2 → 1.2.0.beta1

data/lib/karafka/setup/config.rb

@@ -13,6 +13,7 @@ module Karafka
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
       extend Dry::Configurable
+      extend Callbacks::Config
 
       # Available settings
       # option client_id [String] kafka client_id - used to provide
@@ -21,9 +22,9 @@ module Karafka
       # What backend do we want to use to process messages
       setting :backend, :inline
       # option logger [Instance] logger that we want to use
-      setting :logger, -> { ::Karafka::Logger.instance }
+      setting :logger, -> { ::Karafka::Instrumentation::Logger.instance }
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
-      setting :monitor, -> { ::Karafka::Monitor.instance }
+      setting :monitor, -> { ::Karafka::Instrumentation::Monitor.instance }
       # Mapper used to remap consumer groups ids, so in case users migrate from other tools
       # or they need to maintain their own internal consumer group naming conventions, they
       # can easily do it, replacing the default client_id + consumer name pattern concept
@@ -43,12 +44,24 @@ module Karafka
       # #params_batch will contain params received from Kafka (may be more than 1) so we can
       # process them in batches
       setting :batch_consuming, false
-      # Should we operate in a single controller instance across multiple batches of messages,
-      # from the same partition or should we build a new instance for each incoming batch.
-      # Disabling that can be useful when you want to build a new controller instance for each
-      # incoming batch. It's disabled by default, not to create more objects that needed on
-      # each batch
+      # Should we operate in a single consumer instance across multiple batches of messages,
+      # from the same partition or should we build a new one for each incoming batch.
+      # Disabling that can be useful when you want to create a new consumer instance for each
+      # incoming batch. It's disabled by default, not to create more objects that needed
+      # on each batch
       setting :persistent, true
+      # option shutdown_timeout [Integer, nil] the number of seconds after which Karafka no
+      #   longer wait for the consumers to stop gracefully but instead we force
+      #   terminate everything.
+      # @note Keep in mind, that if your business logic
+      # @note If set to nil, it won't forcefully shutdown the process at all.
+      setting :shutdown_timeout, 60
+      # option params_base_class [Class] base class for params class initialization
+      #   This can be either a Hash or a HashWithIndifferentAccess depending on your
+      #   requirements. Note, that by using HashWithIndifferentAccess, you remove some of the
+      #   performance in favor of convenience. This can be useful especially if you already use
+      #   it with Rails, etc
+      setting :params_base_class, Hash
 
       # option kafka [Hash] - optional - kafka configuration options
       setting :kafka do
@@ -81,6 +94,9 @@ module Karafka
         #   returning messages from the server; if `max_wait_time` is reached, this
         #   is ignored.
         setting :min_bytes, 1
+        # option max_bytes [Integer] the maximum number of bytes to read before returning messages
+        #   from each broker.
+        setting :max_bytes, 10_485_760
         # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
         #   wait before returning data from a single message fetch. By setting this high you also
         #   increase the fetching throughput - and by setting it low you set a bound on latency.
@@ -109,24 +125,33 @@ module Karafka
         setting :socket_timeout, 30
 
         # SSL authentication related settings
-        # option ca_cert [String] SSL CA certificate
+        # option ca_cert [String, nil] SSL CA certificate
         setting :ssl_ca_cert, nil
-        # option ssl_ca_cert_file_path [String] SSL CA certificate file path
+        # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
         setting :ssl_ca_cert_file_path, nil
-        # option ssl_client_cert [String] SSL client certificate
+        # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
+        #   certificate store
+        setting :ssl_ca_certs_from_system, false
+        # option ssl_client_cert [String, nil] SSL client certificate
         setting :ssl_client_cert, nil
-        # option ssl_client_cert_key [String] SSL client certificate password
+        # option ssl_client_cert_key [String, nil] SSL client certificate password
         setting :ssl_client_cert_key, nil
-        # option sasl_gssapi_principal [String] sasl principal
+        # option sasl_gssapi_principal [String, nil] sasl principal
         setting :sasl_gssapi_principal, nil
-        # option sasl_gssapi_keytab [String] sasl keytab
+        # option sasl_gssapi_keytab [String, nil] sasl keytab
         setting :sasl_gssapi_keytab, nil
         # option sasl_plain_authzid [String] The authorization identity to use
         setting :sasl_plain_authzid, ''
-        # option sasl_plain_username [String] The username used to authenticate
+        # option sasl_plain_username [String, nil] The username used to authenticate
         setting :sasl_plain_username, nil
-        # option sasl_plain_password [String] The password used to authenticate
+        # option sasl_plain_password [String, nil] The password used to authenticate
         setting :sasl_plain_password, nil
+        # option sasl_scram_username [String, nil] The username used to authenticate
+        setting :sasl_scram_username, nil
+        # option sasl_scram_password [String, nil] The password used to authenticate
+        setting :sasl_scram_password, nil
+        # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
+        setting :sasl_scram_mechanism, nil
       end
 
       class << self
@@ -134,18 +159,17 @@ module Karafka
         # @yield Runs a block of code providing a config singleton instance to it
         # @yieldparam [Karafka::Setup::Config] Karafka config instance
         def setup
-          configure do |config|
-            yield(config)
-          end
+          configure { |config| yield(config) }
         end
 
         # Everything that should be initialized after the setup
         # Components are in karafka/config directory and are all loaded one by one
         # If you want to configure a next component, please add a proper file to config dir
         def setup_components
-          Configurators::Base.descendants.each do |klass|
-            klass.new(config).setup
-          end
+          [
+            Configurators::Params,
+            Configurators::WaterDrop
+          ].each { |klass| klass.setup(config) }
         end
 
         # Validate config based on ConfigurationSchema

data/lib/karafka/setup/configurators/base.rb

@@ -3,10 +3,13 @@
 module Karafka
   module Setup
     # Configurators module is used to enclose all the external dependencies configurations
+    # upon which Karafka depents
     class Configurators
-      # Karafka has come components that it relies on (like Sidekiq)
+      # Karafka has some components that it relies on (like Sidekiq)
       # We need to configure all of them only when the framework was set up.
       # Any class that descends from this one will be automatically invoked upon setup (after it)
+      # @note This should be used only for internal Karafka dependencies configuration
+      #   End users configuration should go to the after_init block
       # @example Configure an Example class
       #   class ExampleConfigurator < Base
       #     def setup
@@ -15,18 +18,9 @@ module Karafka
       #     end
       #   end
       class Base
-        extend ActiveSupport::DescendantsTracker
-
-        attr_reader :config
-
-        # @param config [Karafka::Config] config instance
-        # @return [Karafka::Config::Base] configurator for a given component
-        def initialize(config)
-          @config = config
-        end
-
+        # @param _config [Karafka::Config] config instance
         # This method needs to be implemented in a subclass
-        def setup
+        def self.setup(_config)
           raise NotImplementedError
         end
       end

data/lib/karafka/setup/configurators/params.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    class Configurators
+      # Karafka::Params::Params are dynamically built based on user defined parent class
+      # so we cannot just require it, we need to initialize it after user is done with
+      # the framework configuration. This is a configurator that does exactly that.
+      class Params < Base
+        # Builds up Karafka::Params::Params class with user defined parent class
+        # @param config [Karafka::Setup::Config] Config we can user to setup things
+        def self.setup(config)
+          return if defined? Karafka::Params::Params
+
+          Karafka::Params.const_set(
+            'Params',
+            Class
+              .new(config.params_base_class)
+              .tap { |klass| klass.include(Karafka::Params::Dsl) }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/water_drop.rb

@@ -6,17 +6,20 @@ module Karafka
       # Class responsible for setting up WaterDrop configuration
       class WaterDrop < Base
         # Sets up a WaterDrop settings
-        def setup
+        # @param config [Karafka::Setup::Config] Config we can user to setup things
+        # @note This will also inject Karafka monitor as a default monitor into WaterDrop,
+        #   so we have the same monitor within whole Karafka framework (same with logger)
+        def self.setup(config)
           ::WaterDrop.setup do |water_config|
             water_config.deliver = true
 
-            Karafka::App.config.to_h.except(:kafka).each do |k, v|
+            config.to_h.except(:kafka).each do |k, v|
               key_assignment = :"#{k}="
               next unless water_config.respond_to?(key_assignment)
               water_config.public_send(key_assignment, v)
             end
 
-            Karafka::App.config.kafka.to_h.each do |k, v|
+            config.kafka.to_h.each do |k, v|
               key_assignment = :"#{k}="
               next unless water_config.kafka.respond_to?(key_assignment)
               water_config.kafka.public_send(key_assignment, v)

data/lib/karafka/setup/dsl.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Dsl for allowing to work with the configuration from the Karafka::App
+    # @note Despite providing methods, everything is still persisted and fetched
+    # from the Karafka::Setup::Config
+    module Dsl
+      # Sets up the whole configuration
+      # @param [Block] block configuration block
+      def setup(&block)
+        Setup::Config.setup(&block)
+        initialize!
+      end
+
+      # @return [Karafka::Config] config instance
+      def config
+        Setup::Config.config
+      end
+    end
+  end
+end

data/lib/karafka/templates/{application_controller.rb.example → application_consumer.rb.example}

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-# Application controller from which all Karafka controllers should inherit
+# Application consumer from which all Karafka consumers should inherit
 # You can rename it if it would conflict with your current code base (in case you're integrating
 # Karafka with other frameworks)
-class ApplicationController < Karafka::BaseController
-end
+ApplicationConsumer = Class.new(Karafka::BaseConsumer)

data/lib/karafka/templates/karafka.rb.example

@@ -21,18 +21,29 @@ class KarafkaApp < Karafka::App
     config.batch_fetching = true
   end
 
+  after_init do |config|
+    # Put here all the things you want to do after the Karafka framework
+    # initialization
+  end
+
+  # Comment out this part if you are not using instrumentation and/or you are not
+  # interested in logging events for certain environments. Since instrumentation
+  # notifications add extra boilerplate, if you want to achieve max performance,
+  # listen to only what you really need for given environment.
+  Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
+
   consumer_groups.draw do
     # topic :example do
-    #   controller ExampleController
+    #   consumer ExampleConsumer
     # end
 
     # consumer_group :bigger_group do
     #   topic :test do
-    #     controller TestController
+    #     consumer TestConsumer
     #   end
     #
     #   topic :test2 do
-    #     controller Test2Controller
+    #     consumer Test2Consumer
     #   end
     # end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.1.2'
+  VERSION = '1.2.0.beta1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-28 00:00:00.000000000 Z
+date: 2018-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -18,14 +18,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -40,6 +40,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: dry-inflector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: dry-monitor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: dry-validation
   requirement: !ruby/object:Gem::Requirement
@@ -116,14 +144,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -144,14 +172,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 1.2.0.beta1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: 1.2.0.beta1
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -182,8 +210,11 @@ files:
 - lib/karafka/app.rb
 - lib/karafka/attributes_map.rb
 - lib/karafka/backends/inline.rb
-- lib/karafka/base_controller.rb
+- lib/karafka/base_consumer.rb
 - lib/karafka/base_responder.rb
+- lib/karafka/callbacks.rb
+- lib/karafka/callbacks/config.rb
+- lib/karafka/callbacks/dsl.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
@@ -191,29 +222,31 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/connection/client.rb
 - lib/karafka/connection/config_adapter.rb
-- lib/karafka/connection/consumer.rb
+- lib/karafka/connection/delegator.rb
 - lib/karafka/connection/listener.rb
-- lib/karafka/connection/processor.rb
-- lib/karafka/controllers/callbacks.rb
-- lib/karafka/controllers/includer.rb
-- lib/karafka/controllers/responders.rb
-- lib/karafka/controllers/single_params.rb
+- lib/karafka/consumers/callbacks.rb
+- lib/karafka/consumers/includer.rb
+- lib/karafka/consumers/responders.rb
+- lib/karafka/consumers/single_params.rb
 - lib/karafka/errors.rb
 - lib/karafka/fetcher.rb
 - lib/karafka/helpers/class_matcher.rb
 - lib/karafka/helpers/config_retriever.rb
 - lib/karafka/helpers/multi_delegator.rb
+- lib/karafka/instrumentation/listener.rb
+- lib/karafka/instrumentation/logger.rb
+- lib/karafka/instrumentation/monitor.rb
 - lib/karafka/loader.rb
-- lib/karafka/logger.rb
-- lib/karafka/monitor.rb
-- lib/karafka/params/params.rb
+- lib/karafka/params/dsl.rb
 - lib/karafka/params/params_batch.rb
 - lib/karafka/parsers/json.rb
 - lib/karafka/patches/dry_configurable.rb
 - lib/karafka/patches/ruby_kafka.rb
+- lib/karafka/persistence/client.rb
 - lib/karafka/persistence/consumer.rb
-- lib/karafka/persistence/controller.rb
+- lib/karafka/persistence/topic.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
 - lib/karafka/responders/topic.rb
@@ -232,9 +265,11 @@ files:
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/configurators/base.rb
+- lib/karafka/setup/configurators/params.rb
 - lib/karafka/setup/configurators/water_drop.rb
+- lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
-- lib/karafka/templates/application_controller.rb.example
+- lib/karafka/templates/application_consumer.rb.example
 - lib/karafka/templates/application_responder.rb.example
 - lib/karafka/templates/karafka.rb.example
 - lib/karafka/version.rb
@@ -254,12 +289,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka