karafka 1.2.0

data/lib/karafka/callbacks/dsl.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Callbacks
+    # App level dsl to define callbacks
+    module Dsl
+      Callbacks::TYPES.each do |callback_type|
+        # Allows us to define a block, that will be executed for a given moment
+        # @param [Block] block that should be executed after the initialization process
+        define_method callback_type do |&block|
+          config.callbacks.send(callback_type).push block
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  # If you want to add/modify command that belongs to CLI, please review all commands
+  # available in cli/ directory inside Karafka source code.
+  #
+  # @note Whole Cli is built using Thor
+  # @see https://github.com/erikhuda/thor
+  class Cli < Thor
+    package_name 'Karafka'
+
+    class << self
+      # Loads all Cli commands into Thor framework
+      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
+      # have any Cli commands available
+      def prepare
+        cli_commands.each do |action|
+          action.bind_to(self)
+        end
+      end
+
+      private
+
+      # @return [Array<Class>] Array with Cli action classes that can be used as commands
+      def cli_commands
+        constants
+          .map! { |object| const_get(object) }
+          .keep_if do |object|
+            object.instance_of?(Class) && (object < Cli::Base)
+          end
+      end
+    end
+  end
+end
+
+# This is kinda trick - since we don't have a autoload and other magic stuff
+# like Rails does, so instead this method allows us to replace currently running
+# console with a new one via Kernel.exec. It will start console with new code loaded
+# Yes we know that it is not turbofast, however it is turbo convinient and small
+#
+# Also - the KARAFKA_CONSOLE is used to detect that we're executing the irb session
+# so this method is only available when the Karafka console is running
+#
+# We skip this because this should exist and be only valid in the console
+# :nocov:
+if ENV['KARAFKA_CONSOLE']
+  # Reloads Karafka irb console session
+  def reload!
+    puts "Reloading...\n"
+    Kernel.exec Karafka::Cli::Console.command
+  end
+end
+# :nocov:

data/lib/karafka/cli/base.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # Base class for all the command that we want to define
+    # This base class provides a nicer interface to Thor and allows to easier separate single
+    # independent commands
+    # In order to define a new command you need to:
+    #   - specify its desc
+    #   - implement call method
+    #
+    # @example Create a dummy command
+    #   class Dummy < Base
+    #     self.desc = 'Dummy command'
+    #
+    #     def call
+    #       puts 'I'm doing nothing!
+    #     end
+    #   end
+    class Base
+      include Thor::Shell
+
+      # We can use it to call other cli methods via this object
+      attr_reader :cli
+
+      # @param cli [Karafka::Cli] current Karafka Cli instance
+      def initialize(cli)
+        @cli = cli
+      end
+
+      # This method should implement proper cli action
+      def call
+        raise NotImplementedError, 'Implement this in a subclass'
+      end
+
+      class << self
+        # Allows to set options for Thor cli
+        # @see https://github.com/erikhuda/thor
+        # @param option Single option details
+        def option(*option)
+          @options ||= []
+          @options << option
+        end
+
+        # Allows to set description of a given cli command
+        # @param desc [String] Description of a given cli command
+        def desc(desc)
+          @desc ||= desc
+        end
+
+        # This method will bind a given Cli command into Karafka Cli
+        # This method is a wrapper to way Thor defines its commands
+        # @param cli_class [Karafka::Cli] Karafka cli_class
+        def bind_to(cli_class)
+          cli_class.desc name, @desc
+
+          (@options || []).each { |option| cli_class.option(*option) }
+
+          context = self
+
+          cli_class.send :define_method, name do |*args|
+            context.new(self).call(*args)
+          end
+        end
+
+        private
+
+        # @return [String] downcased current class name that we use to define name for
+        #   given Cli command
+        # @example for Karafka::Cli::Install
+        #   name #=> 'install'
+        def name
+          to_s.split('::').last.downcase
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/console.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Console Karafka Cli action
+    class Console < Base
+      desc 'Start the Karafka console (short-cut alias: "c")'
+      option aliases: 'c'
+
+      # @return [String] Console executing command
+      # @example
+      #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
+      def self.command
+        envs = [
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
+          'KARAFKA_CONSOLE=true'
+        ]
+        "#{envs.join(' ')} bundle exec irb"
+      end
+
+      # Start the Karafka console
+      def call
+        cli.info
+        exec self.class.command
+      end
+    end
+  end
+end

data/lib/karafka/cli/flow.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Description of topics flow (incoming/outgoing)
+    class Flow < Base
+      desc 'Print application data flow (incoming => outgoing)'
+
+      # Print out all defined routes in alphabetical order
+      def call
+        topics.each do |topic|
+          any_topics = !topic.responder&.topics.nil?
+
+          if any_topics
+            puts "#{topic.name} =>"
+
+            topic.responder.topics.each_value do |responder_topic|
+              features = []
+              features << (responder_topic.required? ? 'always' : 'conditionally')
+              features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')
+
+              print responder_topic.name, "(#{features.join(', ')})"
+            end
+          else
+            puts "#{topic.name} => (nothing)"
+          end
+        end
+      end
+
+      private
+
+      # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
+      def topics
+        Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
+      end
+
+      # Prints a given value with label in a nice way
+      # @param label [String] label describing value
+      # @param value [String] value that should be printed
+      def print(label, value)
+        printf "%-25s %s\n", "  - #{label}:", value
+      end
+    end
+  end
+end

data/lib/karafka/cli/info.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Info Karafka Cli action
+    class Info < Base
+      desc 'Print configuration details and other options of your application'
+
+      # Print configuration details and other options of your application
+      def call
+        config = Karafka::App.config
+
+        info = [
+          "Karafka framework version: #{Karafka::VERSION}",
+          "Application client id: #{config.client_id}",
+          "Backend: #{config.backend}",
+          "Batch fetching: #{config.batch_fetching}",
+          "Batch consuming: #{config.batch_consuming}",
+          "Boot file: #{Karafka.boot_file}",
+          "Environment: #{Karafka.env}",
+          "Kafka seed brokers: #{config.kafka.seed_brokers}"
+        ]
+
+        puts(info.join("\n"))
+      end
+    end
+  end
+end

data/lib/karafka/cli/install.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Install Karafka Cli action
+    class Install < Base
+      desc 'Install all required things for Karafka application in current directory'
+
+      # Directories created by default
+      INSTALL_DIRS = %w[
+        app/consumers
+        app/responders
+        config
+        log
+        tmp/pids
+      ].freeze
+
+      # Where should we map proper files from templates
+      INSTALL_FILES_MAP = {
+        'karafka.rb.example' => Karafka.boot_file.basename,
+        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
+        'application_responder.rb.example' => 'app/responders/application_responder.rb'
+      }.freeze
+
+      # Install all required things for Karafka application in current directory
+      def call
+        INSTALL_DIRS.each do |dir|
+          FileUtils.mkdir_p Karafka.root.join(dir)
+        end
+
+        INSTALL_FILES_MAP.each do |source, target|
+          target = Karafka.root.join(target)
+          next if File.exist?(target)
+
+          source = Karafka.core_root.join("templates/#{source}")
+          FileUtils.cp_r(source, target)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/server.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli < Thor
+    # Server Karafka Cli action
+    class Server < Base
+      desc 'Start the Karafka server (short-cut alias: "s")'
+      option aliases: 's'
+      option :daemon, default: false, type: :boolean, aliases: :d
+      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
+      option :consumer_groups, type: :array, default: nil, aliases: :g
+
+      # Start the Karafka server
+      def call
+        validate!
+
+        puts 'Starting Karafka server'
+        cli.info
+
+        if cli.options[:daemon]
+          FileUtils.mkdir_p File.dirname(cli.options[:pid])
+          daemonize
+        end
+
+        # We assign active topics on a server level, as only server is expected to listen on
+        # part of the topics
+        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+
+        Karafka::Server.run
+      end
+
+      private
+
+      # Checks the server cli configuration
+      # options validations in terms of app setup (topics, pid existence, etc)
+      def validate!
+        result = Schemas::ServerCliOptions.call(cli.options)
+        return if result.success?
+        raise Errors::InvalidConfiguration, result.errors
+      end
+
+      # Detaches current process into background and writes its pidfile
+      def daemonize
+        ::Process.daemon(true)
+        File.open(
+          cli.options[:pid],
+          'w'
+        ) { |file| file.write(::Process.pid) }
+      end
+
+      # Removes a pidfile (if exist)
+      def clean
+        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      end
+    end
+  end
+end

data/lib/karafka/connection/client.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class used as a wrapper around Ruby-Kafka client to simplify additional
+    # features that we provide/might provide in future and to hide the internal implementation
+    class Client
+      extend Forwardable
+
+      def_delegator :kafka_consumer, :seek
+
+      # Creates a queue consumer client that will pull the data from Kafka
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
+      #   we create a client
+      # @return [Karafka::Connection::Client] group consumer that can subscribe to
+      #   multiple topics
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+        Persistence::Client.write(self)
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @note This will yield with raw messages - no preprocessing or reformatting.
+      def fetch_loop
+        settings = ConfigAdapter.consuming(consumer_group)
+
+        if consumer_group.batch_fetching
+          kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
+        else
+          # always yield an array of messages, so we have consistent API (always a batch)
+          kafka_consumer.each_message(*settings) { |message| yield([message]) }
+        end
+      rescue Kafka::ProcessingError => error
+        # If there was an error during consumption, we have to log it, pause current partition
+        # and process other things
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: error.cause
+        )
+        pause(error.topic, error.partition)
+        retry
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => error
+        # rubocop:enable RescueException
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: error
+        )
+        retry
+      end
+
+      # Gracefuly stops topic consumption
+      # @note Stopping running consumers without a really important reason is not recommended
+      #   as until all the consumers are stopped, the server will keep running serving only
+      #   part of the messages
+      def stop
+        @kafka_consumer&.stop
+        @kafka_consumer = nil
+      end
+
+      # Pauses fetching and consumption of a given topic partition
+      # @param topic [String] topic that we want to pause
+      # @param partition [Integer] number partition that we want to pause
+      def pause(topic, partition)
+        settings = ConfigAdapter.pausing(consumer_group)
+        timeout = settings[:timeout]
+        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
+        kafka_consumer.pause(topic, partition, settings)
+      end
+
+      # Marks a given message as consumed and commit the offsets
+      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      def mark_as_consumed(params)
+        kafka_consumer.mark_message_as_processed(params)
+        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
+        # before the automatic triggers have kicked in.
+        kafka_consumer.commit_offsets
+      end
+
+      private
+
+      attr_reader :consumer_group
+
+      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
+      #   that is set up to consume from topics of a given consumer group
+      def kafka_consumer
+        @kafka_consumer ||= kafka.consumer(
+          *ConfigAdapter.consumer(consumer_group)
+        ).tap do |consumer|
+          consumer_group.topics.each do |topic|
+            consumer.subscribe(*ConfigAdapter.subscription(topic))
+          end
+        end
+      rescue Kafka::ConnectionError
+        # If we would not wait it would totally spam log file with failed
+        # attempts if Kafka is down
+        sleep(consumer_group.reconnect_timeout)
+        # We don't log and just reraise - this will be logged
+        # down the road
+        raise
+      end
+
+      # @return [Kafka] returns a Kafka
+      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
+      #   object instance
+      def kafka
+        Kafka.new(*ConfigAdapter.client(consumer_group))
+      end
+    end
+  end
+end