karafka 1.1.1 → 1.2.0

data/lib/karafka/setup/configurators/params.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    class Configurators
+      # Karafka::Params::Params are dynamically built based on user defined parent class
+      # so we cannot just require it, we need to initialize it after user is done with
+      # the framework configuration. This is a configurator that does exactly that.
+      class Params < Base
+        # Builds up Karafka::Params::Params class with user defined parent class
+        # @param config [Karafka::Setup::Config] Config we can user to setup things
+        def self.setup(config)
+          return if defined? Karafka::Params::Params
+
+          Karafka::Params.const_set(
+            'Params',
+            Class
+              .new(config.params_base_class)
+              .tap { |klass| klass.include(Karafka::Params::Dsl) }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/water_drop.rb

@@ -6,17 +6,20 @@ module Karafka
       # Class responsible for setting up WaterDrop configuration
       class WaterDrop < Base
         # Sets up a WaterDrop settings
-        def setup
+        # @param config [Karafka::Setup::Config] Config we can user to setup things
+        # @note This will also inject Karafka monitor as a default monitor into WaterDrop,
+        #   so we have the same monitor within whole Karafka framework (same with logger)
+        def self.setup(config)
           ::WaterDrop.setup do |water_config|
             water_config.deliver = true
 
-            Karafka::App.config.to_h.except(:kafka).each do |k, v|
+            config.to_h.except(:kafka).each do |k, v|
               key_assignment = :"#{k}="
               next unless water_config.respond_to?(key_assignment)
               water_config.public_send(key_assignment, v)
             end
 
-            Karafka::App.config.kafka.to_h.each do |k, v|
+            config.kafka.to_h.each do |k, v|
               key_assignment = :"#{k}="
               next unless water_config.kafka.respond_to?(key_assignment)
               water_config.kafka.public_send(key_assignment, v)

data/lib/karafka/setup/dsl.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Dsl for allowing to work with the configuration from the Karafka::App
+    # @note Despite providing methods, everything is still persisted and fetched
+    # from the Karafka::Setup::Config
+    module Dsl
+      # Sets up the whole configuration
+      # @param [Block] block configuration block
+      def setup(&block)
+        Setup::Config.setup(&block)
+        initialize!
+      end
+
+      # @return [Karafka::Config] config instance
+      def config
+        Setup::Config.config
+      end
+    end
+  end
+end

data/lib/karafka/templates/{application_controller.rb.example → application_consumer.rb.example}

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-# Application controller from which all Karafka controllers should inherit
+# Application consumer from which all Karafka consumers should inherit
 # You can rename it if it would conflict with your current code base (in case you're integrating
 # Karafka with other frameworks)
-class ApplicationController < Karafka::BaseController
-end
+ApplicationConsumer = Class.new(Karafka::BaseConsumer)

data/lib/karafka/templates/karafka.rb.example

@@ -15,24 +15,37 @@ Karafka::Loader.load(Karafka::App.root)
 
 class KarafkaApp < Karafka::App
   setup do |config|
-    config.kafka.seed_brokers = %w( 127.0.0.1:9092 )
+    config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
     config.client_id = 'example_app'
     config.backend = :inline
     config.batch_fetching = true
+    # Uncomment this for Rails app integration
+    # config.logger = Rails.logger
   end
 
+  after_init do |config|
+    # Put here all the things you want to do after the Karafka framework
+    # initialization
+  end
+
+  # Comment out this part if you are not using instrumentation and/or you are not
+  # interested in logging events for certain environments. Since instrumentation
+  # notifications add extra boilerplate, if you want to achieve max performance,
+  # listen to only what you really need for given environment.
+  Karafka.monitor.subscribe(Karafka::Instrumentation::Listener)
+
   consumer_groups.draw do
     # topic :example do
-    #   controller ExampleController
+    #   consumer ExampleConsumer
     # end
 
     # consumer_group :bigger_group do
     #   topic :test do
-    #     controller TestController
+    #     consumer TestConsumer
     #   end
     #
     #   topic :test2 do
-    #     controller Test2Controller
+    #     consumer Test2Consumer
     #   end
     # end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-20 00:00:00.000000000 Z
+date: 2018-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -18,14 +18,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -40,6 +40,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: dry-inflector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: dry-monitor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: dry-validation
   requirement: !ruby/object:Gem::Requirement
@@ -116,14 +144,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.5.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.5.3
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -142,16 +170,16 @@ dependencies:
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '1.2'
 description: Framework used to simplify Apache Kafka based Ruby applications development
 email:
 - maciej@coditsu.io
@@ -182,8 +210,11 @@ files:
 - lib/karafka/app.rb
 - lib/karafka/attributes_map.rb
 - lib/karafka/backends/inline.rb
-- lib/karafka/base_controller.rb
+- lib/karafka/base_consumer.rb
 - lib/karafka/base_responder.rb
+- lib/karafka/callbacks.rb
+- lib/karafka/callbacks/config.rb
+- lib/karafka/callbacks/dsl.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
@@ -191,29 +222,31 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/connection/client.rb
 - lib/karafka/connection/config_adapter.rb
-- lib/karafka/connection/consumer.rb
+- lib/karafka/connection/delegator.rb
 - lib/karafka/connection/listener.rb
-- lib/karafka/connection/processor.rb
-- lib/karafka/controllers/callbacks.rb
-- lib/karafka/controllers/includer.rb
-- lib/karafka/controllers/responders.rb
-- lib/karafka/controllers/single_params.rb
+- lib/karafka/consumers/callbacks.rb
+- lib/karafka/consumers/includer.rb
+- lib/karafka/consumers/responders.rb
+- lib/karafka/consumers/single_params.rb
 - lib/karafka/errors.rb
 - lib/karafka/fetcher.rb
 - lib/karafka/helpers/class_matcher.rb
 - lib/karafka/helpers/config_retriever.rb
 - lib/karafka/helpers/multi_delegator.rb
+- lib/karafka/instrumentation/listener.rb
+- lib/karafka/instrumentation/logger.rb
+- lib/karafka/instrumentation/monitor.rb
 - lib/karafka/loader.rb
-- lib/karafka/logger.rb
-- lib/karafka/monitor.rb
-- lib/karafka/params/params.rb
+- lib/karafka/params/dsl.rb
 - lib/karafka/params/params_batch.rb
 - lib/karafka/parsers/json.rb
 - lib/karafka/patches/dry_configurable.rb
 - lib/karafka/patches/ruby_kafka.rb
+- lib/karafka/persistence/client.rb
 - lib/karafka/persistence/consumer.rb
-- lib/karafka/persistence/controller.rb
+- lib/karafka/persistence/topic.rb
 - lib/karafka/process.rb
 - lib/karafka/responders/builder.rb
 - lib/karafka/responders/topic.rb
@@ -232,9 +265,11 @@ files:
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/configurators/base.rb
+- lib/karafka/setup/configurators/params.rb
 - lib/karafka/setup/configurators/water_drop.rb
+- lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
-- lib/karafka/templates/application_controller.rb.example
+- lib/karafka/templates/application_consumer.rb.example
 - lib/karafka/templates/application_responder.rb.example
 - lib/karafka/templates/karafka.rb.example
 - lib/karafka/version.rb
@@ -259,7 +294,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/karafka/connection/processor.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that consumes messages for which we listen
-    module Processor
-      class << self
-        # Processes messages (does something with them)
-        # It will either schedule or run a proper controller action for messages
-        # @note This should be looped to obtain a constant listening
-        # @note We catch all the errors here, to make sure that none failures
-        #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the thread
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
-        def process(group_id, kafka_messages)
-          # @note We always get messages by topic and partition so we can take topic from the
-          # first one and it will be valid for all the messages
-          # We map from incoming topic name, as it might be namespaced, etc.
-          # @see topic_mapper internal docs
-          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
-          topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-          controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
-            topic.controller.new
-          end
-
-          # Depending on a case (persisted or not) we might use new controller instance per each
-          # batch, or use the same instance for all of them (for implementing buffering, etc)
-          send(
-            topic.batch_consuming ? :process_batch : :process_each,
-            controller,
-            kafka_messages
-          )
-        end
-
-        private
-
-        # Processes whole batch in one request (all at once)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_batch(controller, kafka_messages)
-          controller.params_batch = kafka_messages
-          Karafka.monitor.notice(self, kafka_messages)
-          controller.call
-        end
-
-        # Processes messages one by one (like with std http requests)
-        # @param controller [Karafka::BaseController] base controller descendant
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
-        def process_each(controller, kafka_messages)
-          kafka_messages.each do |kafka_message|
-            # @note This is a simple trick - we just process one after another, but in order
-            # not to handle everywhere both cases (single vs batch), we just "fake" batching with
-            # a single message for each
-            process_batch(controller, [kafka_message])
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/includer.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for controllers
-  module Controllers
-    # Module used to inject functionalities into a given controller class, based on the controller
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is totally not worth having
-    # everything in all the cases all the time
-    module Includer
-      class << self
-        # @param controller_class [Class] controller class, that will get some functionalities
-        #   based on the topic under which it operates
-        def call(controller_class)
-          topic = controller_class.topic
-
-          bind_backend(controller_class, topic)
-          bind_params(controller_class, topic)
-          bind_responders(controller_class, topic)
-        end
-
-        private
-
-        # Figures out backend for a given controller class, based on the topic backend and
-        #   includes it into the controller class
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_backend(controller_class, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          controller_class.include backend
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_params(controller_class, topic)
-          return if topic.batch_consuming
-          controller_class.include SingleParams
-        end
-
-        # Adds responders support for topics and controllers with responders defined for them
-        # @param controller_class [Class] controller class
-        # @param topic [Karafka::Routing::Topic] topic of a controller class
-        def bind_responders(controller_class, topic)
-          return unless topic.responder
-          controller_class.include Responders
-        end
-      end
-    end
-  end
-end

data/lib/karafka/controllers/responders.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Controllers
-    # Feature that allows us to use responders flow in controller
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.notice(self.class, data: data)
-        # @note we build a new instance of responder each time, as a long running (persisted)
-        #   controllers can respond multiple times during the lifecycle
-        topic.responder.new(topic.parser).call(*data)
-      end
-    end
-  end
-end