karafka 0.5.0.3 → 0.6.0.rc1

data/lib/karafka/errors.rb

@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 module Karafka
   # Namespace used to encapsulate all the internal errors of Karafka
   module Errors
     # Base class for all the Karafka internal errors
-    class BaseError < StandardError; end
+    BaseError = Class.new(StandardError)
 
     # Should be raised when we attemp to parse incoming params but parsing fails
     #   If this error (or its descendant) is detected, we will pass the raw message
     #   into params and proceed further
-    class ParserError < BaseError; end
+    ParserError = Class.new(BaseError)
 
     # Raised when router receives topic name which does not correspond with any routes
     # This can only happen in a case when:
@@ -19,41 +21,25 @@ module Karafka
     # In case this happens, you will have to create a temporary route that will allow
     # you to "eat" everything from the Sidekiq queue.
     # @see https://github.com/karafka/karafka/issues/135
-    class NonMatchingRouteError < BaseError; end
-
-    # Raised when we have few controllers(inherited from Karafka::BaseController)
-    #   with the same group name
-    class DuplicatedGroupError < BaseError; end
-
-    # Raised when we have few controllers(inherited from Karafka::BaseController)
-    #   with the same topic name
-    class DuplicatedTopicError < BaseError; end
-
-    # Raised when we want to use topic name that has unsupported characters
-    class InvalidTopicName < BaseError; end
-
-    # Raised when we want to use group name that has unsupported characters
-    class InvalidGroupName < BaseError; end
+    NonMatchingRouteError = Class.new(BaseError)
 
     # Raised when application does not have ApplicationWorker or other class that directly
     # inherits from Karafka::BaseWorker
-    class BaseWorkerDescentantMissing < BaseError; end
+    BaseWorkerDescentantMissing = Class.new(BaseError)
 
     # Raised when we want to use #respond_with in controllers but we didn't define
     # (and we couldn't find) any appropriate responder for a given controller
-    class ResponderMissing < BaseError; end
-
-    # Raised when we want to use #respond_to in responders with a topic that we didn't register
-    class UnregisteredTopic < BaseError; end
+    ResponderMissing = Class.new(BaseError)
 
-    # Raised when we send more than one message to a single topic but we didn't allow that when
-    # we were registering topic in a responder
-    class TopicMultipleUsage < BaseError; end
-
-    # Raised when we didn't use a topic that was defined as non-optional (required)
-    class UnusedResponderRequiredTopic < BaseError; end
+    # Raised when we don't use or use responder not in the way it expected to based on the
+    # topics usage definitions
+    InvalidResponderUsage = Class.new(BaseError)
 
     # Raised when configuration doesn't match with validation schema
-    class InvalidConfiguration < BaseError; end
+    InvalidConfiguration = Class.new(BaseError)
+
+    # Raised when processing messages in batches but still want to use #params instead of
+    # #params_batch
+    ParamsMethodUnavailable = Class.new(BaseError)
   end
 end

data/lib/karafka/fetcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Class used to run the Karafka consumer and handle shutting down, restarting etc
   # @note Creating multiple fetchers will result in having multiple connections to the same
@@ -8,7 +10,7 @@ module Karafka
     # so we don't have to terminate them
     def fetch_loop
       futures = listeners.map do |listener|
-        listener.future.public_send(:fetch_loop, consumer)
+        listener.future.public_send(:fetch_loop, processor)
       end
 
       futures.map(&:value)
@@ -24,16 +26,16 @@ module Karafka
 
     # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
     def listeners
-      @listeners ||= App.routes.map do |route|
-        Karafka::Connection::Listener.new(route)
+      @listeners ||= App.consumer_groups.map do |consumer_group|
+        Karafka::Connection::Listener.new(consumer_group)
       end
     end
 
-    # @return [Proc] proc that should be processed when a message arrives
-    # @yieldparam message [Kafka::FetchedMessage] message from kafka (raw one)
-    def consumer
-      lambda do |message|
-        Karafka::Connection::Consumer.new.consume(message)
+    # @return [Proc] proc that should be processed when a messages arrive
+    # @yieldparam messages [Array<Kafka::FetchedMessage>] messages from kafka (raw)
+    def processor
+      lambda do |consumer_group_id, messages|
+        Karafka::Connection::MessagesProcessor.process(consumer_group_id, messages)
       end
     end
   end

data/lib/karafka/helpers/class_matcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   module Helpers
     # Class used to autodetect corresponding classes that are internally inside Karafka framework

data/lib/karafka/helpers/config_retriever.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # A helper method that allows us to build methods that try to get a given
+    # attribute from its instance value and if it fails, will fallback to
+    # the default config or config.kafka value for a given attribute.
+    # It is used to simplify the checkings.
+    # @note Worth noticing, that the value might be equal to false, so even
+    #   then we need to return it. That's why we check for nil?
+    # @example Define config retried attribute for start_from_beginning
+    # class Test
+    #   extend Karafka::Helpers::ConfigRetriever
+    #   config_retriever_for :start_from_beginning
+    # end
+    #
+    # Test.new.start_from_beginning #=> false
+    # test_instance = Test.new
+    # test_instance.start_from_beginning = true
+    # test_instance.start_from_beginning #=> true
+    module ConfigRetriever
+      # Builds proper methods for setting and retrieving (with fallback) given attribute value
+      # @param attribute [Symbol] attribute name based on which we will build
+      #   accessor with fallback
+      def config_retriever_for(attribute)
+        attr_writer attribute unless method_defined? :"#{attribute}="
+
+        # Don't redefine if we already have accessor for a given element
+        return if method_defined? attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+          return current_value unless current_value.nil?
+
+          value = if Karafka::App.config.respond_to?(attribute)
+                    Karafka::App.config.public_send(attribute)
+                  else
+                    Karafka::App.config.kafka.public_send(attribute)
+                  end
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Module containing classes and methods that provide some additional functionalities
   module Helpers

data/lib/karafka/loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Loader for requiring all the files in a proper order
   # Some files needs to be required before other, so it will
@@ -5,7 +7,7 @@ module Karafka
   # any other.
   class Loader
     # Order in which we want to load app files
-    DIRS = %w(
+    DIRS = %w[
       config/initializers
       lib
       app/helpers
@@ -20,7 +22,7 @@ module Karafka
       app/controllers
       app/aspects
       app
-    ).freeze
+    ].freeze
 
     # Will load files in a proper order (based on DIRS)
     # @param [String] root path from which we want to start

data/lib/karafka/logger.rb

@@ -1,7 +1,11 @@
+# frozen_string_literal: true
+
 module Karafka
   # Default logger for Event Delegator
   # @note It uses ::Logger features - providing basic logging
   class Logger < ::Logger
+    include Singleton
+
     # Map containing informations about log level for given environment
     ENV_MAP = {
       'production' => ::Logger::ERROR,
@@ -11,42 +15,39 @@ module Karafka
       default: ::Logger::INFO
     }.freeze
 
-    class << self
-      # Returns a logger instance with appropriate settings, log level and environment
-      def instance
-        ensure_dir_exists
-        instance = new(target)
-        instance.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
-        instance
-      end
-
-      private
-
-      # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
-      #   to which we will be writtng logs
-      # We use this approach to log stuff to file and to the STDOUT at the same time
-      def target
-        Karafka::Helpers::MultiDelegator
-          .delegate(:write, :close)
-          .to(STDOUT, file)
-      end
-
-      # Makes sure the log directory exists
-      def ensure_dir_exists
-        dir = File.dirname(log_path)
-        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-      end
-
-      # @return [Pathname] Path to a file to which we should log
-      def log_path
-        Karafka::App.root.join("log/#{Karafka.env}.log")
-      end
-
-      # @return [File] file to which we want to write our logs
-      # @note File is being opened in append mode ('a')
-      def file
-        File.open(log_path, 'a')
-      end
+    # Creates a new instance of logger ensuring that it has a place to write to
+    def initialize(*_args)
+      ensure_dir_exists
+      super(target)
+      self.level = ENV_MAP[Karafka.env] || ENV_MAP[:default]
+    end
+
+    private
+
+    # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+    #   to which we will be writtng logs
+    # We use this approach to log stuff to file and to the STDOUT at the same time
+    def target
+      Karafka::Helpers::MultiDelegator
+        .delegate(:write, :close)
+        .to(STDOUT, file)
+    end
+
+    # Makes sure the log directory exists
+    def ensure_dir_exists
+      dir = File.dirname(log_path)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+    end
+
+    # @return [Pathname] Path to a file to which we should log
+    def log_path
+      @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
+    end
+
+    # @return [File] file to which we want to write our logs
+    # @note File is being opened in append mode ('a')
+    def file
+      @file ||= File.open(log_path, 'a')
     end
   end
 end

data/lib/karafka/monitor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Monitor is used to hookup external monitoring services to monitor how Karafka works
   # It provides a standarized API for checking incoming messages/enqueueing etc
@@ -53,7 +55,7 @@ module Karafka
     def caller_exceptions_map
       @caller_exceptions_map ||= {
         error: [
-          Karafka::Connection::Consumer,
+          Karafka::Connection::MessagesProcessor,
           Karafka::Connection::Listener,
           Karafka::Params::Params
         ],

data/lib/karafka/params/interchanger.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   module Params
     # Interchangers allow us to format/encode/pack data that is being send to perform_async

data/lib/karafka/params/params.rb

@@ -1,59 +1,52 @@
+# frozen_string_literal: true
+
 module Karafka
   # Params namespace encapsulating all the logic that is directly related to params handling
   module Params
     # Class-wrapper for hash with indifferent access with additional lazy loading feature
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using parser until we execute our logic inside worker. That way we can operate with
-    # heavy-parsing data without slowing down the whole application. If we won't use
-    # params in before_enqueue (or if we don't us before_enqueue at all), it will make
-    # Karafka faster, because it will pass data as it is directly to Sidekiq
+    # heavy-parsing data without slowing down the whole application.
     class Params < HashWithIndifferentAccess
+      # Kafka::FetchedMessage attributes that we want to use inside of params
+      KAFKA_MESSAGE_ATTRIBUTES = %i[
+        topic
+        value
+        partition
+        offset
+        key
+      ].freeze
+
       class << self
         # We allow building instances only via the #build method
-        private_class_method :new
 
-        # @param message [Karafka::Connection::Message, Hash] message that we get out of Kafka
+        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
         #   in case of building params inside main Karafka process in
-        #   Karafka::Connection::Consumer, or a hash when we retrieve data from Sidekiq
-        # @param controller [Karafka::BaseController] Karafka's base controllers descendant
-        #   instance that wants to use params
+        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
+        # @param parser [Class] parser class that we will use to unparse data
         # @return [Karafka::Params::Params] Karafka params object not yet used parser for
         #   retrieving data that we've got from Kafka
         # @example Build params instance from a hash
-        #   Karafka::Params::Params.build({ key: 'value' }, DataController.new) #=> params object
-        # @example Build params instance from a Karafka::Connection::Message object
-        #   Karafka::Params::Params.build(message, IncomingController.new) #=> params object
-        def build(message, controller)
+        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
+        # @example Build params instance from a Kafka::FetchedMessage object
+        #   Karafka::Params::Params.build(message) #=> params object
+        def build(message, parser)
           # Hash case happens inside workers
           if message.is_a?(Hash)
-            defaults(controller).merge!(message)
+            new(parser: parser).merge!(message)
           else
-            # This happens inside Karafka::Connection::Consumer
-            defaults(controller).merge!(
+            # This happens inside Kafka::FetchedMessagesProcessor
+            new(
+              parser: parser,
               parsed: false,
-              received_at: Time.now,
-              content: message.content
-            )
+              received_at: Time.now
+            ).tap do |instance|
+              KAFKA_MESSAGE_ATTRIBUTES.each do |attribute|
+                instance[attribute] = message.send(attribute)
+              end
+            end
           end
         end
-
-        private
-
-        # @param controller [Karafka::BaseController] Karafka's base controllers descendant
-        #   instance that wants to use params
-        # @return [Karafka::Params::Params] freshly initialized only with default values object
-        #   that can be populated with incoming data
-        def defaults(controller)
-          # We initialize some default values that will be used both in Karafka main process and
-          # inside workers
-          new(
-            controller: controller.class,
-            worker: controller.worker,
-            parser: controller.parser,
-            topic: controller.topic,
-            responder: controller.responder
-          )
-        end
       end
 
       # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
@@ -63,7 +56,7 @@ module Karafka
       def retrieve
         return self if self[:parsed]
 
-        merge!(parse(delete(:content)))
+        merge!(parse(delete(:value)))
       end
 
       # Overwritten merge! method - it behaves differently for keys that are the same in our hash
@@ -85,16 +78,16 @@ module Karafka
 
       private
 
-      # @param content [String] Raw data that we want to parse using controller's parser
+      # @param value [String] Raw data that we want to parse using controller's parser
       # @note If something goes wrong, it will return raw data in a hash with a message key
       # @return [Hash] parsed data or a hash with message key containing raw data if something
       #   went wrong during parsing
-      def parse(content)
-        self[:parser].parse(content)
+      def parse(value)
+        self[:parser].parse(value)
         # We catch both of them, because for default JSON - we use JSON parser directly
       rescue ::Karafka::Errors::ParserError => e
         Karafka.monitor.notice_error(self.class, e)
-        return { message: content }
+        raise e
       ensure
         self[:parsed] = true
       end

data/lib/karafka/params/params_batch.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Params batch represents a set of messages received from Kafka.
+    # @note Params internally are lazy loaded before first use. That way we can skip parsing
+    #   process if we have before_enqueue that rejects some incoming messages without using params
+    #   It can be also used when handling really heavy data (in terms of parsing). Without direct
+    #   usage outside of worker scope, it will pass raw data into sidekiq, so we won't use Karafka
+    #   working time to parse this data. It will happen only in the worker (where it can take time)
+    #   that way Karafka will be able to process data really quickly. On the other hand, if we
+    #   decide to use params somewhere before it hits worker logic, it won't parse it again in
+    #   the worker - it will use already loaded data and pass it to Redis
+    class ParamsBatch
+      include Enumerable
+
+      # Builds up a params batch based on raw kafka messages
+      # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
+      # @param topic_parser [Class] topic parser for unparsing messages values
+      def initialize(messages_batch, topic_parser)
+        @params_batch = messages_batch.map do |message|
+          Karafka::Params::Params.build(message, topic_parser)
+        end
+      end
+
+      # @yieldparam [Karafka::Params::Params] each parsed and loaded params instance
+      # @note Invocation of this method will cause loading and parsing each param after another.
+      #   If you want to get access without parsing, please access params_batch directly
+      def each
+        @params_batch.each { |param| yield(param.retrieve) }
+      end
+
+      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
+      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
+      #   be parsed
+      def parsed
+        each(&:itself)
+      end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
+      def to_a
+        @params_batch
+      end
+    end
+  end
+end