karafka 0.5.0.3 → 0.6.0.rc1

data/Rakefile

@@ -1,17 +1,8 @@
+# frozen_string_literal: true
+
 require 'bundler'
 require 'rake'
-require 'polishgeeks-dev-tools'
-
-PolishGeeks::DevTools.setup do |config|
-  config.brakeman = false
-  config.haml_lint = false
-end
-
-desc 'Self check using polishgeeks-dev-tools'
-task :check do
-  PolishGeeks::DevTools::Runner.new.execute(
-    PolishGeeks::DevTools::Logger.new
-  )
-end
+require 'rspec/core/rake_task'
 
-task default: :check
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/karafka.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
@@ -8,28 +10,32 @@ Gem::Specification.new do |spec|
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w( maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com )
+  spec.email       = %w[maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com]
   spec.homepage    = 'https://github.com/karafka/karafka'
-  spec.summary     = %q{ Ruby based framework for working with Apache Kafka }
-  spec.description = %q{ Framework used to simplify Apache Kafka based Ruby applications development }
+  spec.summary     = 'Ruby based framework for working with Apache Kafka'
+  spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_development_dependency 'bundler', '~> 1.2'
-
-  spec.add_dependency 'ruby-kafka', '= 0.3.17'
+  spec.add_dependency 'ruby-kafka', '>= 0.4'
   spec.add_dependency 'sidekiq', '>= 4.2'
-  spec.add_dependency 'worker-glass', '~> 0.2'
-  spec.add_dependency 'celluloid', '~> 0.17'
+  spec.add_dependency 'celluloid'
   spec.add_dependency 'envlogic', '~> 1.0'
-  spec.add_dependency 'waterdrop', '~> 0.3.2.4'
-  spec.add_dependency 'rake', '~> 11.3'
+  spec.add_dependency 'waterdrop', '>= 0.4'
+  spec.add_dependency 'rake', '>= 11.3'
   spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'activesupport', '~> 5.0'
-  spec.add_dependency 'dry-validation', '~> 0.10.6'
+  spec.add_dependency 'activesupport', '>= 5.0'
+  spec.add_dependency 'dry-validation', '~> 0.11'
   spec.add_dependency 'dry-configurable', '~> 0.7'
+  spec.add_dependency 'yajl-ruby', '>= 1.3.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.2'
+  spec.add_development_dependency 'rspec', '>= 3.6'
+  spec.add_development_dependency 'simplecov', '>= 0.14'
+  spec.add_development_dependency 'byebug'
+
   spec.required_ruby_version = '>= 2.3.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.require_paths = %w( lib )
+  spec.require_paths = %w[lib]
 end

data/lib/karafka.rb

@@ -1,4 +1,6 @@
-%w(
+# frozen_string_literal: true
+
+%w[
   rake
   ostruct
   rubygems
@@ -11,10 +13,10 @@
   logger
   kafka
   sidekiq
-  worker_glass
   envlogic
   thor
   fileutils
+  yajl
   dry-configurable
   dry-validation
   active_support/callbacks
@@ -23,9 +25,10 @@
   active_support/descendants_tracker
   active_support/inflector
   karafka/loader
+  karafka/setup/config
   karafka/status
-  karafka/routing/route
-).each { |lib| require lib }
+  karafka/routing/router
+].each(&method(:require))
 
 # Karafka library
 module Karafka

data/lib/karafka/app.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # App class
   class App
@@ -10,7 +12,7 @@ module Karafka
       end
 
       # Sets up all the internal components and bootstrap whole app
-      # We need to know details about routes in order to setup components,
+      # We need to know details about consumers in order to setup components,
       # that's why we don't setup them after std setup is done
       # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
       #   doesn't match with ConfigurationSchema
@@ -24,8 +26,8 @@ module Karafka
         Setup::Config.config
       end
 
-      # @return [Karafka::Routing::Builder] routes builder instance
-      def routes
+      # @return [Karafka::Routing::Builder] consumers builder instance
+      def consumer_groups
         Routing::Builder.instance
       end
 
@@ -36,9 +38,11 @@ module Karafka
       end
 
       # Methods that should be delegated to Karafka module
-      %i(
-        root env logger monitor
-      ).each do |delegated|
+      %i[
+        root
+        env
+        logger monitor
+      ].each do |delegated|
         define_method(delegated) do
           Karafka.public_send(delegated)
         end

data/lib/karafka/attributes_map.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
+  # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
+  # is on consumer, subscription and consumption levels. In order to maintain an order
+  # in managing those settings, this module was created. It contains details on what setting
+  # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
+  # setting it and sending it forward
+  # @note Settings presented here cover all the settings that are being used across Karafka
+  module AttributesMap
+    class << self
+      # What settings should go where in ruby-kafka
+      # @note All other settings will be passed to Kafka.new method invocation.
+      #   All elements in this hash are just edge cases
+      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
+      def config_adapter
+        {
+          consumer: %i[
+            session_timeout offset_commit_interval offset_commit_threshold
+            offset_retention_time heartbeat_interval
+          ],
+          subscription: %i[start_from_beginning max_bytes_per_partition],
+          consuming: %i[min_bytes max_wait_time],
+          # All the options that are under kafka config namespace, but are not used
+          # directly with kafka api, but from the Karafka user perspective, they are
+          # still related to kafka. They should not be proxied anywhere
+          ignored: %i[reconnect_timeout]
+        }
+      end
+
+      # @return [Array<Symbol>] properties that can be set on a per topic level
+      def topic
+        (config_adapter[:subscription] + %i[
+          inline_processing
+          name
+          worker
+          parser
+          interchanger
+          responder
+          batch_processing
+        ]).uniq
+      end
+
+      # @return [Array<Symbol>] properties that can be set on a per consumer group level
+      # @note Note that there are settings directly extracted from the config kafka namespace
+      #   I did this that way, so I won't have to repeat same setting keys over and over again
+      #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
+      #   to add it to our configuration class and it will be handled automatically.
+      def consumer_group
+        # @note We don't ignore the config_adapter[:ignored] values as they should be ignored
+        #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
+        ignored_settings = config_adapter[:subscription]
+        defined_settings = config_adapter.values.flatten
+        karafka_settings = %i[batch_consuming topic_mapper]
+        # This is a drity and bad hack of dry-configurable to get keys before setting values
+        dynamically_proxied = Karafka::Setup::Config
+                              ._settings
+                              .find { |s| s.name == :kafka }
+                              .value
+                              .instance_variable_get('@klass').settings
+
+        (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
+      end
+    end
+  end
+end

data/lib/karafka/base_controller.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Karafka module namespace
 module Karafka
   # Base controller from which all Karafka controllers should inherit
@@ -62,17 +64,10 @@ module Karafka
     # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
     define_callbacks :schedule
 
-    # This will be set based on routing settings
-    # From 0.4 a single controller can handle multiple topics jobs
-    # All the attributes are taken from route
-    Karafka::Routing::Route::ATTRIBUTES.each do |attr|
-      attr_reader attr
-
-      define_method(:"#{attr}=") do |new_attr_value|
-        instance_variable_set(:"@#{attr}", new_attr_value)
-        @params[attr] = new_attr_value if @params
-      end
-    end
+    # Each controller instance is always bind to a single topic. We don't place it on a class
+    # level because some programmers use same controller for multiple topics
+    attr_accessor :topic
+    attr_accessor :params_batch
 
     class << self
       # Creates a callback that will be executed before scheduling to Sidekiq
@@ -91,30 +86,39 @@ module Karafka
       end
     end
 
-    # Creates lazy loaded params object
+    # Creates lazy loaded params batch object
     # @note Until first params usage, it won't parse data at all
-    # @param message [Karafka::Connection::Message, Hash] message with raw content or a hash
-    #   from Sidekiq that allows us to build params.
-    def params=(message)
-      @params = Karafka::Params::Params.build(message, self)
+    # @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
+    #   content (from Kafka) or messages inside a hash (from Sidekiq, etc)
+    # @return [Karafka::Params::ParamsBatch] lazy loaded params batch
+    def params_batch=(messages)
+      @params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
+    end
+
+    # @return [Karafka::Params::Params] params instance for non batch processed controllers
+    # @raise [Karafka::Errors::ParamsMethodUnavailable] raised when we try to use params
+    #   method in a batch_processed controller
+    def params
+      raise Karafka::Errors::ParamsMethodUnavailable if topic.batch_processing
+      params_batch.first
     end
 
     # Executes the default controller flow, runs callbacks and if not halted
-    # will schedule a perform task in sidekiq
+    # will schedule a call task in sidekiq
     def schedule
       run_callbacks :schedule do
-        inline_mode ? perform_inline : perform_async
+        topic.inline_processing ? call_inline : call_async
       end
     end
 
-    # @return [Hash] hash with all controller details - it works similar to #params method however
-    #   it won't parse data so it will return unparsed details about controller and its parameters
-    # @example Get data about ctrl
-    #   ctrl.to_h #=> { "worker"=>WorkerClass, "parsed"=>false, "content"=>"{}" }
-    def to_h
-      @params
+    # @note We want to leave the #perform method as a public API, but just in case we will do some
+    #   pre or post processing we use call method instead of directly executing #perform
+    def call
+      perform
     end
 
+    private
+
     # Method that will perform business logic on data received from Kafka
     # @note This method needs bo be implemented in a subclass. We stub it here as a failover if
     #   someone forgets about it or makes on with typo
@@ -122,21 +126,10 @@ module Karafka
       raise NotImplementedError, 'Implement this in a subclass'
     end
 
-    private
-
-    # @return [Karafka::Params::Params] Karafka params that is a hash with indifferent access
-    # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have before_enqueue that rejects some incoming messages without using params
-    #   It can be also used when handling really heavy data (in terms of parsing). Without direct
-    #   usage outside of worker scope, it will pass raw data into sidekiq, so we won't use Karafka
-    #   working time to parse this data. It will happen only in the worker (where it can take time)
-    #   that way Karafka will be able to process data really quickly. On the other hand, if we
-    #   decide to use params somewhere before it hits worker logic, it won't parse it again in
-    #   the worker - it will use already loaded data and pass it to Redis
-    # @note Invokation of this method will cause load all the data into params object. If you want
-    #   to get access without parsing, please access @params directly
-    def params
-      @params.retrieve
+    # @return [Karafka::BaseResponder] responder instance if defined
+    # @return [nil] nil if no responder for this controller
+    def responder
+      @responder ||= topic.responder&.new(topic.parser)
     end
 
     # Responds with given data using given responder. This allows us to have a similar way of
@@ -147,30 +140,27 @@ module Karafka
     #   but we still try to use this method
     def respond_with(*data)
       raise(Errors::ResponderMissing, self.class) unless responder
-
       Karafka.monitor.notice(self.class, data: data)
-      responder.new(parser).call(*data)
+      responder.call(*data)
     end
 
     # Executes perform code immediately (without enqueuing)
     # @note Despite the fact, that workers won't be used, we still initialize all the
     #   classes and other framework elements
-    def perform_inline
-      Karafka.monitor.notice(self.class, to_h)
-      perform
+    def call_inline
+      Karafka.monitor.notice(self.class, params_batch)
+      call
     end
 
     # Enqueues the execution of perform method into a worker.
     # @note Each worker needs to have a class #perform_async method that will allow us to pass
-    #   parameters into it. We always pass topic as a first argument and this request params
+    #   parameters into it. We always pass topic as a first argument and this request params_batch
     #   as a second one (we pass topic to be able to build back the controller in the worker)
-    def perform_async
-      Karafka.monitor.notice(self.class, to_h)
-      # We use @params directly (instead of #params) because of lazy loading logic that is behind
-      # it. See Karafka::Params::Params class for more details about that
-      worker.perform_async(
-        topic,
-        interchanger.load(@params)
+    def call_async
+      Karafka.monitor.notice(self.class, params_batch)
+      topic.worker.perform_async(
+        topic.id,
+        topic.interchanger.load(params_batch.to_a)
       )
     end
   end

data/lib/karafka/base_responder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Base responder from which all Karafka responders should inherit
   # Similar to Rails responders concept. It allows us to design flow from one app to another
@@ -68,7 +70,7 @@ module Karafka
       # @param options [Hash] hash with optional configuration details
       def topic(topic_name, options = {})
         self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options)
+        topic_obj = Responders::Topic.new(topic_name, options.merge(registered: true))
         self.topics[topic_obj.name] = topic_obj
       end
 
@@ -79,6 +81,9 @@ module Karafka
       # @example Send user data with a responder (uses default Karafka::Parsers::Json parser)
       #   UsersCreatedResponder.call(@created_user)
       def call(*data)
+        # Just in case there were no topics defined for a responder, we initialize with
+        # empty hash not to handle a nil case
+        self.topics ||= {}
         new.call(*data)
       end
     end
@@ -109,6 +114,30 @@ module Karafka
 
     private
 
+    # Checks if we met all the topics requirements. It will fail if we didn't send a message to
+    # a registered required topic, etc.
+    def validate!
+      registered_topics = self.class.topics.map do |name, topic|
+        topic.to_h.merge!(
+          usage_count: messages_buffer[name]&.count || 0
+        )
+      end
+
+      used_topics = messages_buffer.map do |name, usage|
+        topic = self.class.topics[name] || Responders::Topic.new(name, registered: false)
+        topic.to_h.merge!(usage_count: usage.count)
+      end
+
+      result = Karafka::Schemas::ResponderUsage.call(
+        registered_topics: registered_topics,
+        used_topics: used_topics
+      )
+
+      return if result.success?
+
+      raise Karafka::Errors::InvalidResponderUsage, result.errors
+    end
+
     # Method that needs to be implemented in a subclass. It should handle responding
     #   on registered topics
     # @raise [NotImplementedError] This method needs to be implemented in a subclass
@@ -129,19 +158,6 @@ module Karafka
       messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
     end
 
-    # Checks if we met all the topics requirements. It will fail if we didn't send a message to
-    # a registered required topic, etc.
-    def validate!
-      used_topics = messages_buffer.map do |key, data_elements|
-        Array.new(data_elements.count) { key }
-      end
-
-      Responders::UsageValidator.new(
-        self.class.topics || {},
-        used_topics.flatten
-      ).validate!
-    end
-
     # Takes all the messages from the buffer and delivers them one by one
     # @note This method is executed after the validation, so we're sure that
     #   what we send is legit and it will go to a proper topics

data/lib/karafka/base_worker.rb

@@ -1,40 +1,25 @@
+# frozen_string_literal: true
+
 module Karafka
   # Worker wrapper for Sidekiq workers
   class BaseWorker
     include Sidekiq::Worker
 
-    attr_accessor :params, :topic
-
     # Executes the logic that lies in #perform Karafka controller method
-    # @param topic [String] Topic that we will use to route to a proper controller
-    # @param params [Hash] params hash that we use to build Karafka params object
-    def perform(topic, params)
-      self.topic = topic
-      self.params = params
-      Karafka.monitor.notice(self.class, controller.to_h)
-      controller.perform
-    end
-
-    # What action should be taken when perform method fails
-    # @param topic [String] Topic bthat we will use to route to a proper controller
-    # @param params [Hash] params hash that we use to build Karafka params object
-    def after_failure(topic, params)
-      self.topic = topic
-      self.params = params
-
-      return unless controller.respond_to?(:after_failure)
-
-      Karafka.monitor.notice(self.class, controller.to_h)
-      controller.after_failure
+    # @param topic_id [String] Unique topic id that we will use to find a proper topic
+    # @param params_batch [Array] Array with messages batch
+    def perform(topic_id, params_batch)
+      Karafka.monitor.notice(self.class, params_batch)
+      controller(topic_id, params_batch).call
     end
 
     private
 
     # @return [Karafka::Controller] descendant of Karafka::BaseController that matches the topic
-    #   with params assigned already (controller is ready to use)
-    def controller
-      @controller ||= Karafka::Routing::Router.new(topic).build.tap do |ctrl|
-        ctrl.params = ctrl.interchanger.parse(params)
+    #   with params_batch assigned already (controller is ready to use)
+    def controller(topic_id, params_batch)
+      @controller ||= Karafka::Routing::Router.build(topic_id).tap do |ctrl|
+        ctrl.params_batch = ctrl.topic.interchanger.parse(params_batch)
       end
     end
   end