karafka 1.1.0

data/lib/karafka/cli/console.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Console Karafka Cli action
+    class Console < Base
+      desc 'Start the Karafka console (short-cut alias: "c")'
+      option aliases: 'c'
+
+      # @return [String] Console executing command
+      # @example
+      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+      def self.command
+        envs = [
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+          'KARAFKA_CONSOLE=true'
+        ]
+        "#{envs.join(' ')} bundle exec irb"
+      end
+
+      # Start the Karafka console
+      def call
+        cli.info
+        exec self.class.command
+      end
+    end
+  end
+end

data/lib/karafka/cli/flow.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Description of topics flow (incoming/outgoing)
+    class Flow < Base
+      desc 'Print application data flow (incoming => outgoing)'
+
+      # Print out all defined routes in alphabetical order
+      def call
+        topics.each do |topic|
+          any_topics = !topic.responder&.topics.nil?
+
+          if any_topics
+            puts "#{topic.name} =>"
+
+            topic.responder.topics.each_value do |responder_topic|
+              features = []
+              features << (responder_topic.required? ? 'always' : 'conditionally')
+              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
+
+              print responder_topic.name, "(#{features.join(', ')})"
+            end
+          else
+            puts "#{topic.name} => (nothing)"
+          end
+        end
+      end
+
+      private
+
+      # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
+      def topics
+        Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
+      end
+
+      # Prints a given value with label in a nice way
+      # @param label [String] label describing value
+      # @param value [String] value that should be printed
+      def print(label, value)
+        printf "%-25s %s\n", "  - #{label}:", value
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Info Karafka Cli action
+    class Info < Base
+      desc 'Print configuration details and other options of your application'
+
+      # Print configuration details and other options of your application
+      def call
+        config = Karafka::App.config
+
+        info = [
+          "Karafka framework version: #{Karafka::VERSION}",
+          "Application client id: #{config.client_id}",
+          "Backend: #{config.backend}",
+          "Batch fetching: #{config.batch_fetching}",
+          "Batch consuming: #{config.batch_consuming}",
+          "Boot file: #{Karafka.boot_file}",
+          "Environment: #{Karafka.env}",
+          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+        ]
+
+        puts(info.join("\n"))
+      end
+    end
+  end
+end

data/lib/karafka/cli/install.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Install Karafka Cli action
+    class Install < Base
+      desc 'Install all required things for Karafka application in current directory'
+
+      # Directories created by default
+      INSTALL_DIRS = %w[
+        app/models
+        app/controllers
+        app/responders
+        config
+        log
+        tmp/pids
+      ].freeze
+
+      # Where should we map proper files from templates
+      INSTALL_FILES_MAP = {
+        'karafka.rb.example' => Karafka.boot_file.basename,
+        'application_controller.rb.example' => 'app/controllers/application_controller.rb',
+        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+      }.freeze
+
+      # Install all required things for Karafka application in current directory
+      def call
+        INSTALL_DIRS.each do |dir|
+          FileUtils.mkdir_p Karafka.root.join(dir)
+        end
+
+        INSTALL_FILES_MAP.each do |source, target|
+          target = Karafka.root.join(target)
+          next if File.exist?(target)
+
+          source = Karafka.core_root.join("templates/#{source}")
+          FileUtils.cp_r(source, target)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Server Karafka Cli action
+    class Server < Base
+      desc 'Start the Karafka server (short-cut alias: "s")'
+      option aliases: 's'
+      option :daemon, default: false, type: :boolean, aliases: :d
+      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
+      option :consumer_groups, type: :array, default: nil, aliases: :g
+
+      # Start the Karafka server
+      def call
+        validate!
+
+        puts 'Starting Karafka server'
+        cli.info
+
+        if cli.options[:daemon]
+          FileUtils.mkdir_p File.dirname(cli.options[:pid])
+          daemonize
+        end
+
+        # We assign active topics on a server level, as only server is expected to listen on
+        # part of the topics
+        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+
+        # After we fork, we can boot celluloid again
+        Karafka::Server.run
+      end
+
+      private
+
+      # Checks the server cli configuration
+      # options validations in terms of app setup (topics, pid existence, etc)
+      def validate!
+        result = Schemas::ServerCliOptions.call(cli.options)
+        return if result.success?
+        raise Errors::InvalidConfiguration, result.errors
+      end
+
+      # Detaches current process into background and writes its pidfile
+      def daemonize
+        ::Process.daemon(true)
+        File.open(
+          cli.options[:pid],
+          'w'
+        ) { |file| file.write(::Process.pid) }
+      end
+
+      # Removes a pidfile (if exist)
+      def clean
+        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      end
+    end
+  end
+end

data/lib/karafka/connection/config_adapter.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related to Kafka connection
+  module Connection
+    # Mapper used to convert our internal settings into ruby-kafka settings
+    # Since ruby-kafka has more and more options and there are few "levels" on which
+    # we have to apply them (despite the fact, that in Karafka you configure all of it
+    # in one place), we have to remap it into what ruby-kafka driver requires
+    # @note The good thing about Kafka.new method is that it ignores all options that
+    #   do nothing. So we don't have to worry about injecting our internal settings
+    #   into the client and breaking stuff
+    module ConfigAdapter
+      class << self
+        # Builds all the configuration settings for Kafka.new method
+        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required by Kafka.new method
+        def client(_consumer_group)
+          # This one is a default that takes all the settings except special
+          # cases defined in the map
+          settings = {
+            logger: ::Karafka.logger,
+            client_id: ::Karafka::App.config.client_id
+          }
+
+          kafka_configs.each do |setting_name, setting_value|
+            # All options for config adapter should be ignored as we're just interested
+            # in what is left, as we want to pass all the options that are "typical"
+            # and not listed in the config_adapter special cases mapping. All the values
+            # from the config_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
+
+            settings[setting_name] = setting_value
+          end
+
+          sanitize(settings)
+        end
+
+        # Builds all the configuration settings for kafka#consumer method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required by Kafka#consumer method
+        def consumer(consumer_group)
+          settings = { group_id: consumer_group.id }
+          settings = fetch_for(:consumer, consumer_group, settings)
+          sanitize(settings)
+        end
+
+        # Builds all the configuration settings for kafka consumer consume_each_batch and
+        #   consume_each_message methods
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required by
+        #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
+        def consuming(consumer_group)
+          settings = {
+            automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+          }
+          sanitize(fetch_for(:consuming, consumer_group, settings))
+        end
+
+        # Builds all the configuration settings for kafka consumer#subscribe method
+        # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
+        # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
+        def subscription(topic)
+          settings = fetch_for(:subscription, topic)
+          [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
+        end
+
+        # Builds all the configuration settings required by kafka consumer#pause method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required to pause kafka consumer
+        def pausing(consumer_group)
+          { timeout: consumer_group.pause_timeout }
+        end
+
+        private
+
+        # Fetches proper settings for a given map namespace
+        # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param route_layer [Object] route topic or consumer group
+        # @param preexisting_settings [Hash] hash with some preexisting settings that might have
+        #   been loaded in a different way
+        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
+          kafka_configs.each_key do |setting_name|
+            # Ignore settings that are not related to our namespace
+            next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            # Ignore settings that are already initialized
+            # In case they are in preexisting settings fetched differently
+            next if preexisting_settings.keys.include?(setting_name)
+            # Fetch all the settings from a given layer object. Objects can handle the fallback
+            # to the kafka settings, so
+            preexisting_settings[setting_name] = route_layer.send(setting_name)
+          end
+
+          preexisting_settings
+        end
+
+        # Removes nil containing keys from the final settings so it can use Kafkas driver
+        #   defaults for those
+        # @param settings [Hash] settings that may contain nil values
+        # @return [Hash] settings without nil using keys (non of karafka options should be nil)
+        def sanitize(settings)
+          settings.reject { |_key, value| value.nil? }
+        end
+
+        # @return [Hash] Kafka config details as a hash
+        def kafka_configs
+          ::Karafka::App.config.kafka.to_h
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/consumer.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class used as a wrapper around Ruby-Kafka to simplify additional
+    # features that we provide/might provide in future and to hide the internal implementation
+    class Consumer
+      # Creates a queue consumer that will pull the data from Kafka
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
+      #   we create a client
+      # @return [Karafka::Connection::Consumer] group consumer that can subscribe to
+      #   multiple topics
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+        Persistence::Consumer.write(self)
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @note This will yield with raw messages - no preprocessing or reformatting.
+      def fetch_loop
+        send(
+          consumer_group.batch_fetching ? :consume_each_batch : :consume_each_message
+        ) { |messages| yield(messages) }
+      rescue Kafka::ProcessingError => e
+        # If there was an error during consumption, we have to log it, pause current partition
+        # and process other things
+        Karafka.monitor.notice_error(self.class, e.cause)
+        pause(e.topic, e.partition)
+        retry
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        # rubocop:enable RescueException
+        Karafka.monitor.notice_error(self.class, e)
+        retry
+      end
+
+      # Gracefuly stops topic consumption
+      # @note Stopping running consumers without a really important reason is not recommended
+      #   as until all the consumers are stopped, the server will keep running serving only
+      #   part of the messages
+      def stop
+        @kafka_consumer&.stop
+        @kafka_consumer = nil
+      end
+
+      # Pauses fetching and consumption of a given topic partition
+      # @param topic [String] topic that we want to pause
+      # @param partition [Integer] number partition that we want to pause
+      def pause(topic, partition)
+        settings = ConfigAdapter.pausing(consumer_group)
+        timeout = settings[:timeout]
+        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
+        kafka_consumer.pause(topic, partition, settings)
+      end
+
+      # Marks a given message as consumed and commit the offsets
+      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      def mark_as_consumed(params)
+        kafka_consumer.mark_message_as_processed(params)
+        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
+        # before the automatic triggers have kicked in.
+        kafka_consumer.commit_offsets
+      end
+
+      private
+
+      attr_reader :consumer_group
+
+      # Consumes messages from Kafka in batches
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      def consume_each_batch
+        kafka_consumer.each_batch(
+          ConfigAdapter.consuming(consumer_group)
+        ) do |batch|
+          yield(batch.messages)
+        end
+      end
+
+      # Consumes messages from Kafka one by one
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      def consume_each_message
+        kafka_consumer.each_message(
+          ConfigAdapter.consuming(consumer_group)
+        ) do |message|
+          #   always yield an array of messages, so we have consistent API (always a batch)
+          yield([message])
+        end
+      end
+
+      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
+      #   that is set up to consume from topics of a given consumer group
+      def kafka_consumer
+        @kafka_consumer ||= kafka.consumer(
+          ConfigAdapter.consumer(consumer_group)
+        ).tap do |consumer|
+          consumer_group.topics.each do |topic|
+            consumer.subscribe(*ConfigAdapter.subscription(topic))
+          end
+        end
+      rescue Kafka::ConnectionError
+        # If we would not wait it would totally spam log file with failed
+        # attempts if Kafka is down
+        sleep(consumer_group.reconnect_timeout)
+        # We don't log and just reraise - this will be logged
+        # down the road
+        raise
+      end
+
+      # @return [Kafka] returns a Kafka
+      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
+      #   object instance
+      def kafka
+        Kafka.new(ConfigAdapter.client(consumer_group))
+      end
+    end
+  end
+end