karafka 0.5.0.3 → 0.6.0.rc1

data/lib/karafka/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   # If you want to add/modify command that belongs to CLI, please review all commands

data/lib/karafka/cli/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   class Cli < Thor
     # Base class for all the command that we want to define

data/lib/karafka/cli/console.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   class Cli
@@ -10,7 +12,11 @@ module Karafka
       # @example
       #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
       def self.command
-        "KARAFKA_CONSOLE=true bundle exec irb -r #{Karafka.boot_file}"
+        envs = [
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+          'KARAFKA_CONSOLE=true'
+        ]
+        "#{envs.join(' ')} bundle exec irb"
       end
 
       # Start the Karafka console

data/lib/karafka/cli/flow.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   class Cli
@@ -7,32 +9,30 @@ module Karafka
 
       # Print out all defined routes in alphabetical order
       def call
-        routes.each do |route|
-          any_topics = !route.responder&.topics.nil?
+        topics.each do |topic|
+          any_topics = !topic.responder&.topics.nil?
 
           if any_topics
-            puts "#{route.topic} =>"
+            puts "#{topic.name} =>"
 
-            route.responder.topics.each do |_name, topic|
+            topic.responder.topics.each do |_name, responder_topic|
               features = []
-              features << (topic.required? ? 'always' : 'conditionally')
-              features << (topic.multiple_usage? ? 'one or more' : 'exactly once')
+              features << (responder_topic.required? ? 'always' : 'conditionally')
+              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
 
-              print topic.name, "(#{features.join(', ')})"
+              print responder_topic.name, "(#{features.join(', ')})"
             end
           else
-            puts "#{route.topic} => (nothing)"
+            puts "#{topic.name} => (nothing)"
           end
         end
       end
 
       private
 
-      # @return [Array<Karafka::Routing::Route>] all routes sorted in alphabetical order
-      def routes
-        Karafka::App.routes.sort do |route1, route2|
-          route1.topic <=> route2.topic
-        end
+      # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
+      def topics
+        Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
       end
 
       # Prints a given value with label in a nice way

data/lib/karafka/cli/info.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   class Cli
@@ -11,13 +13,14 @@ module Karafka
 
         info = [
           "Karafka framework version: #{Karafka::VERSION}",
-          "Application name: #{config.name}",
-          "Inline mode: #{config.inline_mode}",
-          "Batch mode: #{config.batch_mode}",
+          "Application client id: #{config.client_id}",
+          "Inline processing: #{config.inline_processing}",
+          "Batch consuming: #{config.batch_consuming}",
+          "Batch processing: #{config.batch_processing}",
           "Number of threads: #{config.concurrency}",
           "Boot file: #{Karafka.boot_file}",
           "Environment: #{Karafka.env}",
-          "Kafka hosts: #{config.kafka.hosts}",
+          "Kafka seed brokers: #{config.kafka.seed_brokers}",
           "Redis: #{config.redis.to_h}"
         ]
 

data/lib/karafka/cli/install.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   class Cli
@@ -6,7 +8,7 @@ module Karafka
       desc 'Install all required things for Karafka application in current directory'
 
       # Directories created by default
-      INSTALL_DIRS = %w(
+      INSTALL_DIRS = %w[
         app/models
         app/controllers
         app/responders
@@ -14,12 +16,11 @@ module Karafka
         config
         log
         tmp/pids
-      ).freeze
+      ].freeze
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'app.rb.example' => Karafka.boot_file.basename,
-        'config.ru.example' => 'config.ru',
         'sidekiq.yml.example' => 'config/sidekiq.yml.example',
         'application_worker.rb.example' => 'app/workers/application_worker.rb',
         'application_controller.rb.example' => 'app/controllers/application_controller.rb',

data/lib/karafka/cli/server.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   class Cli
@@ -26,7 +28,7 @@ module Karafka
         end
 
         # Remove pidfile on shutdown
-        ObjectSpace.define_finalizer('string', proc { send(:clean) })
+        ObjectSpace.define_finalizer(String.new, proc { send(:clean) })
 
         # After we fork, we can boot celluloid again
         Karafka::Server.run

data/lib/karafka/cli/worker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Karafka framework Cli
   class Cli

data/lib/karafka/connection/config_adapter.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related to Kafka connection
+  module Connection
+    # Mapper used to convert our internal settings into ruby-kafka settings
+    # Since ruby-kafka has more and more options and there are few "levels" on which
+    # we have to apply them (despite the fact, that in Karafka you configure all of it
+    # in one place), we have to remap it into what ruby-kafka driver requires
+    # @note The good thing about Kafka.new method is that it ignores all options that
+    #   do nothing. So we don't have to worry about injecting our internal settings
+    #   into the client and breaking stuff
+    module ConfigAdapter
+      class << self
+        # Builds all the configuration settings for Kafka.new method
+        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required by Kafka.new method
+        def client(_consumer_group)
+          # This one is a default that takes all the settings except special
+          # cases defined in the map
+          settings = {
+            logger: ::Karafka.logger,
+            client_id: ::Karafka::App.config.client_id
+          }
+
+          kafka_configs.each do |setting_name, setting_value|
+            # All options for config adapter should be ignored as we're just interested
+            # in what is left, as we want to pass all the options that are "typical"
+            # and not listed in the config_adapter special cases mapping. All the values
+            # from the config_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
+
+            settings[setting_name] = setting_value
+          end
+
+          sanitize(settings)
+        end
+
+        # Builds all the configuration settings for kafka#consumer method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required by Kafka#consumer method
+        def consumer(consumer_group)
+          settings = { group_id: consumer_group.id }
+
+          kafka_configs.each do |setting_name, setting_value|
+            next unless AttributesMap.config_adapter[:consumer].include?(setting_name)
+            next if settings.keys.include?(setting_name)
+            settings[setting_name] = setting_value
+          end
+
+          sanitize(settings)
+        end
+
+        # Builds all the configuration settings for kafka consumer consume_each_batch and
+        #   consume_each_message methods
+        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required by
+        #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
+        def consuming(_consumer_group)
+          settings = {}
+
+          kafka_configs.each do |setting_name, setting_value|
+            next unless AttributesMap.config_adapter[:consuming].include?(setting_name)
+            next if settings.keys.include?(setting_name)
+            settings[setting_name] = setting_value
+          end
+
+          sanitize(settings)
+        end
+
+        # Builds all the configuration settings for kafka consumer#subscribe method
+        # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
+        # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
+        def subscription(topic)
+          settings = { start_from_beginning: topic.start_from_beginning }
+
+          kafka_configs.each do |setting_name, setting_value|
+            next unless AttributesMap.config_adapter[:subscription].include?(setting_name)
+            next if settings.keys.include?(setting_name)
+            settings[setting_name] = setting_value
+          end
+
+          [topic.name, sanitize(settings)]
+        end
+
+        private
+
+        # Removes nil containing keys from the final settings so it can use Kafkas driver
+        #   defaults for those
+        # @param settings [Hash] settings that may contain nil values
+        # @return [Hash] settings without nil using keys (non of karafka options should be nil)
+        def sanitize(settings)
+          settings.reject { |_key, value| value.nil? }
+        end
+
+        # @return [Hash] Kafka config details as a hash
+        def kafka_configs
+          ::Karafka::App.config.kafka.to_h
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   module Connection
     # A single listener that listens to incoming messages from a single route
@@ -9,16 +11,18 @@ module Karafka
 
       execute_block_on_receiver :fetch_loop
 
-      attr_reader :route
+      attr_reader :consumer_group
 
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
+      #   on what topics and with what settings should we listen
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(route)
-        @route = route
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
       end
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Karafka::BaseController] base controller descendant
-      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
+      # @yieldparam [String] consumer group id
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
       # @note This will yield with a raw message - no preprocessing or reformatting
       # @note We catch all the errors here, so they don't affect other listeners (or this one)
       #   so we will be able to listen and consume other incoming messages.
@@ -27,25 +31,25 @@ module Karafka
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
       def fetch_loop(block)
-        topic_consumer.fetch_loop do |raw_message|
-          block.call(raw_message)
+        messages_consumer.fetch_loop do |raw_messages|
+          block.call(consumer_group.id, raw_messages)
         end
         # This is on purpose - see the notes for this method
         # rubocop:disable RescueException
       rescue Exception => e
         # rubocop:enable RescueException
         Karafka.monitor.notice_error(self.class, e)
-        @topic_consumer&.stop
-        retry if @topic_consumer
+        @messages_consumer&.stop
+        retry if @messages_consumer
       end
 
       private
 
-      # @return [Karafka::Connection::TopicConsumer] wrapped kafka consumer for a given topic
+      # @return [Karafka::Connection::MessagesConsumer] wrapped kafka consumer for a given topic
       #   consumption
       # @note It adds consumer into Karafka::Server consumers pool for graceful shutdown on exit
-      def topic_consumer
-        @topic_consumer ||= TopicConsumer.new(@route).tap do |consumer|
+      def messages_consumer
+        @messages_consumer ||= MessagesConsumer.new(consumer_group).tap do |consumer|
           Karafka::Server.consumers << consumer if Karafka::Server.consumers
         end
       end

data/lib/karafka/connection/messages_consumer.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class used as a wrapper around Ruby-Kafka to simplify additional
+    # features that we provide/might provide in future
+    class MessagesConsumer
+      # Creates a queue consumer that will pull the data from Kafka
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
+      #   we create a client
+      # @return [Karafka::Connection::MessagesConsumer] group consumer that can subscribe to
+      #   multiple topics
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @note This will yield with raw messages - no preprocessing or reformatting.
+      def fetch_loop
+        send(
+          consumer_group.batch_consuming ? :consume_each_batch : :consume_each_message
+        ) do |messages|
+          yield(messages)
+        end
+      end
+
+      # Gracefuly stops topic consumption
+      def stop
+        @kafka_consumer&.stop
+        @kafka_consumer = nil
+      end
+
+      private
+
+      attr_reader :consumer_group
+
+      # Consumes messages from Kafka in batches
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      def consume_each_batch
+        kafka_consumer.each_batch(
+          ConfigAdapter.consuming(consumer_group)
+        ) do |batch|
+          yield(batch.messages)
+        end
+      end
+
+      # Consumes messages from Kafka one by one
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      def consume_each_message
+        kafka_consumer.each_message(
+          ConfigAdapter.consuming(consumer_group)
+        ) do |message|
+          #   always yield an array of messages, so we have consistent API (always a batch)
+          yield([message])
+        end
+      end
+
+      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
+      #   that is set up to consume from topics of a given consumer group
+      def kafka_consumer
+        @kafka_consumer ||= kafka.consumer(
+          ConfigAdapter.consumer(consumer_group)
+        ).tap do |consumer|
+          consumer_group.topics.each do |topic|
+            consumer.subscribe(*ConfigAdapter.subscription(topic))
+          end
+        end
+      rescue Kafka::ConnectionError
+        # If we would not wait it would totally spam log file with failed
+        # attempts if Kafka is down
+        sleep(consumer_group.reconnect_timeout)
+        # We don't log and just reraise - this will be logged
+        # down the road
+        raise
+      end
+
+      # @return [Kafka] returns a Kafka
+      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
+      #   object instance
+      def kafka
+        Kafka.new(ConfigAdapter.client(consumer_group))
+      end
+    end
+  end
+end

data/lib/karafka/connection/messages_processor.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that consumes messages for which we listen
+    module MessagesProcessor
+      class << self
+        # Processes messages (does something with them)
+        # It will either schedule or run a proper controller action for messages
+        # @note This should be looped to obtain a constant listening
+        # @note We catch all the errors here, to make sure that none failures
+        #   for a given consumption will affect other consumed messages
+        #   If we wouldn't catch it, it would propagate up until killing the Celluloid actor
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
+        def process(group_id, kafka_messages)
+          # @note We always get messages by topic and partition so we can take topic from the
+          # first one and it will be valid for all the messages
+          # We map from incoming topic name, as it might be namespaced, etc.
+          # @see topic_mapper internal docs
+          mapped_topic = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
+          # @note We search based on the topic id - that is a combination of group id and
+          # topic name
+          controller = Karafka::Routing::Router.build("#{group_id}_#{mapped_topic}")
+          handler = controller.topic.batch_processing ? :process_batch : :process_each
+
+          send(handler, controller, mapped_topic, kafka_messages)
+          # This is on purpose - see the notes for this method
+          # rubocop:disable RescueException
+        rescue Exception => e
+          # rubocop:enable RescueException
+          Karafka.monitor.notice_error(self, e)
+        end
+
+        private
+
+        # Processes whole batch in one request (all at once)
+        # @param controller [Karafka::BaseController] base controller descendant
+        # @param mapped_topic [String] mapped topic name
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
+        def process_batch(controller, mapped_topic, kafka_messages)
+          messages_batch = kafka_messages.map do |kafka_message|
+            # Since we support topic mapping (for Kafka providers that require namespaces)
+            # we have to overwrite topic with our mapped topic version
+            # @note For the default mapper, it will be the same as topic
+            # @note We have to use instance_variable_set, as the Kafka::FetchedMessage does not
+            #   provide attribute writers
+            kafka_message.instance_variable_set(:'@topic', mapped_topic)
+            kafka_message
+          end
+
+          controller.params_batch = messages_batch
+
+          Karafka.monitor.notice(self, messages_batch)
+
+          controller.schedule
+        end
+
+        # Processes messages one by one (like with std http requests)
+        # @param controller [Karafka::BaseController] base controller descendant
+        # @param mapped_topic [String] mapped topic name
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
+        def process_each(controller, mapped_topic, kafka_messages)
+          kafka_messages.each do |kafka_message|
+            # @note This is a simple trick - we just process one after another, but in order
+            # not to handle everywhere both cases (single vs batch), we just "fake" batching with
+            # a single message for each
+            process_batch(controller, mapped_topic, [kafka_message])
+          end
+        end
+      end
+    end
+  end
+end