karafka 1.2.11

data/lib/karafka/callbacks/dsl.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Callbacks
+    # App level dsl to define callbacks
+    module Dsl
+      Callbacks::TYPES.each do |callback_type|
+        # Allows us to define a block, that will be executed for a given moment
+        # @param [Block] block that should be executed after the initialization process
+        define_method callback_type do |&block|
+          config.callbacks.send(callback_type).push block
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  # If you want to add/modify command that belongs to CLI, please review all commands
+  # available in cli/ directory inside Karafka source code.
+  #
+  # @note Whole Cli is built using Thor
+  # @see https://github.com/erikhuda/thor
+  class Cli < Thor
+    package_name 'Karafka'
+
+    class << self
+      # Loads all Cli commands into Thor framework
+      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
+      # have any Cli commands available
+      def prepare
+        cli_commands.each do |action|
+          action.bind_to(self)
+        end
+      end
+
+      private
+
+      # @return [Array<Class>] Array with Cli action classes that can be used as commands
+      def cli_commands
+        constants
+          .map! { |object| const_get(object) }
+          .keep_if do |object|
+            object.instance_of?(Class) && (object < Cli::Base)
+          end
+      end
+    end
+  end
+end
+
+# This is kinda trick - since we don't have a autoload and other magic stuff
+# like Rails does, so instead this method allows us to replace currently running
+# console with a new one via Kernel.exec. It will start console with new code loaded
+# Yes we know that it is not turbofast, however it is turbo convinient and small
+#
+# Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
+# so this method is only available when the Karafka console is running
+#
+# We skip this because this should exist and be only valid in the console
+# :nocov:
+if ENV['KARAFKA_CONSOLE']
+  # Reloads Karafka irb console session
+  def reload!
+    puts "Reloading...\n"
+    Kernel.exec Karafka::Cli::Console.command
+  end
+end
+# :nocov:

data/lib/karafka/cli/base.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # Base class for all the command that we want to define
+    # This base class provides a nicer interface to Thor and allows to easier separate single
+    # independent commands
+    # In order to define a new command you need to:
+    #   - specify its desc
+    #   - implement call method
+    #
+    # @example Create a dummy command
+    #   class Dummy < Base
+    #     self.desc = 'Dummy command'
+    #
+    #     def call
+    #       puts 'I'm doing nothing!
+    #     end
+    #   end
+    class Base
+      include Thor::Shell
+
+      # We can use it to call other cli methods via this object
+      attr_reader :cli
+
+      # @param cli [Karafka::Cli] current Karafka Cli instance
+      def initialize(cli)
+        @cli = cli
+      end
+
+      # This method should implement proper cli action
+      def call
+        raise NotImplementedError, 'Implement this in a subclass'
+      end
+
+      class << self
+        # Allows to set options for Thor cli
+        # @see https://github.com/erikhuda/thor
+        # @param option Single option details
+        def option(*option)
+          @options ||= []
+          @options << option
+        end
+
+        # Allows to set description of a given cli command
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
+        end
+
+        # This method will bind a given Cli command into Karafka Cli
+        # This method is a wrapper to way Thor defines its commands
+        # @param cli_class [Karafka::Cli] Karafka cli_class
+        def bind_to(cli_class)
+          cli_class.desc name, @desc
+
+          (@options || []).each { |option| cli_class.option(*option) }
+
+          context = self
+
+          cli_class.send :define_method, name do |*args|
+            context.new(self).call(*args)
+          end
+        end
+
+        private
+
+        # @return [String] downcased current class name that we use to define name for
+        #   given Cli command
+        # @example for Karafka::Cli::Install
+        #   name #=> 'install'
+        def name
+          to_s.split('::').last.downcase
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/console.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Console Karafka Cli action
+    class Console < Base
+      desc 'Start the Karafka console (short-cut alias: "c")'
+      option aliases: 'c'
+
+      # @return [String] Console executing command
+      # @example
+      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+      def self.command
+        envs = [
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+          'KARAFKA_CONSOLE=true'
+        ]
+        "#{envs.join(' ')} bundle exec irb"
+      end
+
+      # Start the Karafka console
+      def call
+        cli.info
+        exec self.class.command
+      end
+    end
+  end
+end

data/lib/karafka/cli/flow.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Description of topics flow (incoming/outgoing)
+    class Flow < Base
+      desc 'Print application data flow (incoming => outgoing)'
+
+      # Print out all defined routes in alphabetical order
+      def call
+        topics.each do |topic|
+          any_topics = !topic.responder&.topics.nil?
+
+          if any_topics
+            puts "#{topic.name} =>"
+
+            topic.responder.topics.each_value do |responder_topic|
+              features = []
+              features << (responder_topic.required? ? 'always' : 'conditionally')
+              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
+
+              print responder_topic.name, "(#{features.join(', ')})"
+            end
+          else
+            puts "#{topic.name} => (nothing)"
+          end
+        end
+      end
+
+      private
+
+      # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
+      def topics
+        Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
+      end
+
+      # Prints a given value with label in a nice way
+      # @param label [String] label describing value
+      # @param value [String] value that should be printed
+      def print(label, value)
+        printf "%-25s %s\n", "  - #{label}:", value
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Info Karafka Cli action
+    class Info < Base
+      desc 'Print configuration details and other options of your application'
+
+      # Print configuration details and other options of your application
+      def call
+        config = Karafka::App.config
+
+        info = [
+          "Karafka framework version: #{Karafka::VERSION}",
+          "Application client id: #{config.client_id}",
+          "Backend: #{config.backend}",
+          "Batch fetching: #{config.batch_fetching}",
+          "Batch consuming: #{config.batch_consuming}",
+          "Boot file: #{Karafka.boot_file}",
+          "Environment: #{Karafka.env}",
+          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+        ]
+
+        puts(info.join("\n"))
+      end
+    end
+  end
+end

data/lib/karafka/cli/install.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Install Karafka Cli action
+    class Install < Base
+      desc 'Install all required things for Karafka application in current directory'
+
+      # Directories created by default
+      INSTALL_DIRS = %w[
+        app/consumers
+        app/responders
+        config
+        log
+        tmp/pids
+      ].freeze
+
+      # Where should we map proper files from templates
+      INSTALL_FILES_MAP = {
+        'karafka.rb.example' => Karafka.boot_file.basename,
+        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+      }.freeze
+
+      # Install all required things for Karafka application in current directory
+      def call
+        INSTALL_DIRS.each do |dir|
+          FileUtils.mkdir_p Karafka.root.join(dir)
+        end
+
+        INSTALL_FILES_MAP.each do |source, target|
+          target = Karafka.root.join(target)
+          next if File.exist?(target)
+
+          source = Karafka.core_root.join("templates/#{source}")
+          FileUtils.cp_r(source, target)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Server Karafka Cli action
+    class Server < Base
+      desc 'Start the Karafka server (short-cut alias: "s")'
+      option aliases: 's'
+      option :daemon, default: false, type: :boolean, aliases: :d
+      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
+      option :consumer_groups, type: :array, default: nil, aliases: :g
+
+      # Start the Karafka server
+      def call
+        validate!
+
+        puts 'Starting Karafka server'
+        cli.info
+
+        if cli.options[:daemon]
+          FileUtils.mkdir_p File.dirname(cli.options[:pid])
+          daemonize
+        end
+
+        # We assign active topics on a server level, as only server is expected to listen on
+        # part of the topics
+        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+
+        Karafka::Server.run
+      end
+
+      private
+
+      # Checks the server cli configuration
+      # options validations in terms of app setup (topics, pid existence, etc)
+      def validate!
+        result = Schemas::ServerCliOptions.call(cli.options)
+        return if result.success?
+        raise Errors::InvalidConfiguration, result.errors
+      end
+
+      # Detaches current process into background and writes its pidfile
+      def daemonize
+        ::Process.daemon(true)
+        File.open(
+          cli.options[:pid],
+          'w'
+        ) { |file| file.write(::Process.pid) }
+      end
+
+      # Removes a pidfile (if exist)
+      def clean
+        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      end
+    end
+  end
+end

data/lib/karafka/connection/api_adapter.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related to Kafka connection
+  module Connection
+    # Mapper used to convert our internal settings into ruby-kafka settings based on their
+    # API requirements.
+    # Since ruby-kafka has more and more options and there are few "levels" on which
+    # we have to apply them (despite the fact, that in Karafka you configure all of it
+    # in one place), we have to remap it into what ruby-kafka driver requires
+    # @note The good thing about Kafka.new method is that it ignores all options that
+    #   do nothing. So we don't have to worry about injecting our internal settings
+    #   into the client and breaking stuff
+    module ApiAdapter
+      class << self
+        # Builds all the configuration settings for Kafka.new method
+        # @return [Array<Hash>] Array with all the client arguments including hash with all
+        #   the settings required by Kafka.new method
+        # @note We return array, so we can inject any arguments we want, in case of changes in the
+        #   raw driver
+        def client
+          # This one is a default that takes all the settings except special
+          # cases defined in the map
+          settings = {
+            logger: ::Karafka.logger,
+            client_id: ::Karafka::App.config.client_id
+          }
+
+          kafka_configs.each do |setting_name, setting_value|
+            # All options for config adapter should be ignored as we're just interested
+            # in what is left, as we want to pass all the options that are "typical"
+            # and not listed in the api_adapter special cases mapping. All the values
+            # from the api_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
+
+            settings[setting_name] = setting_value
+          end
+
+          settings_hash = sanitize(settings)
+
+          # Normalization for the way Kafka::Client accepts arguments from  0.5.3
+          [settings_hash.delete(:seed_brokers), settings_hash]
+        end
+
+        # Builds all the configuration settings for kafka#consumer method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        #   the settings required by Kafka#consumer
+        def consumer(consumer_group)
+          settings = { group_id: consumer_group.id }
+          settings = fetch_for(:consumer, consumer_group, settings)
+          [sanitize(settings)]
+        end
+
+        # Builds all the configuration settings for kafka consumer consume_each_batch and
+        #   consume_each_message methods
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] Array with all the arguments required by consuming method
+        #   including hash with all the settings required by
+        #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
+        def consumption(consumer_group)
+          [
+            sanitize(
+              fetch_for(
+                :consumption,
+                consumer_group,
+                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+              )
+            )
+          ]
+        end
+
+        # Builds all the configuration settings for kafka consumer#subscribe method
+        # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
+        # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
+        def subscribe(topic)
+          settings = fetch_for(:subscribe, topic)
+          [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
+        end
+
+        # Builds all the configuration settings required by kafka consumer#pause method
+        # @param topic [String] topic that we want to pause
+        # @param partition [Integer] number partition that we want to pause
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array] array with all the details required to pause kafka consumer
+        def pause(topic, partition, consumer_group)
+          [
+            Karafka::App.config.topic_mapper.outgoing(topic),
+            partition,
+            { timeout: consumer_group.pause_timeout }
+          ]
+        end
+
+        # Remaps topic details taking the topic mapper feature into consideration.
+        # @param params [Karafka::Params::Params] params instance
+        # @return [Array] array with all the details needed by ruby-kafka to mark message
+        #   as processed
+        # @note When default empty topic mapper is used, no need for any conversion as the
+        #   internal and external format are exactly the same
+        def mark_message_as_processed(params)
+          # Majority of non heroku users don't use custom topic mappers. No need to change
+          # anything when it is a default mapper that does not change anything
+          return [params] if Karafka::App.config.topic_mapper == Karafka::Routing::TopicMapper
+
+          # @note We don't use tap as it is around 13% slower than non-dup version
+          dupped = params.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
+          [dupped]
+        end
+
+        private
+
+        # Fetches proper settings for a given map namespace
+        # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param route_layer [Object] route topic or consumer group
+        # @param preexisting_settings [Hash] hash with some preexisting settings that might have
+        #   been loaded in a different way
+        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
+          kafka_configs.each_key do |setting_name|
+            # Ignore settings that are not related to our namespace
+            next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
+            # Ignore settings that are already initialized
+            # In case they are in preexisting settings fetched differently
+            next if preexisting_settings.key?(setting_name)
+            # Fetch all the settings from a given layer object. Objects can handle the fallback
+            # to the kafka settings, so
+            preexisting_settings[setting_name] = route_layer.send(setting_name)
+          end
+
+          preexisting_settings
+        end
+
+        # Removes nil containing keys from the final settings so it can use Kafkas driver
+        #   defaults for those
+        # @param settings [Hash] settings that may contain nil values
+        # @return [Hash] settings without nil using keys (non of karafka options should be nil)
+        def sanitize(settings)
+          settings.reject { |_key, value| value.nil? }
+        end
+
+        # @return [Hash] Kafka config details as a hash
+        def kafka_configs
+          ::Karafka::App.config.kafka.to_h
+        end
+      end
+    end
+  end
+end