karafka 1.2.11

data/lib/karafka/schemas/server_cli_options.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Schema for validating correctness of the server cli command options
+    # We validate some basics + the list of consumer_groups on which we want to use, to make
+    # sure that all of them are defined, plus that a pidfile does not exist
+    ServerCliOptions = Dry::Validation.Schema do
+      configure do
+        option :consumer_groups
+
+        def self.messages
+          super.merge(
+            en: {
+              errors: {
+                consumer_groups_inclusion: 'Unknown consumer group.',
+                pid_existence: 'Pidfile already exists.'
+              }
+            }
+          )
+        end
+      end
+
+      optional(:pid).filled(:str?)
+      optional(:daemon).filled(:bool?)
+      optional(:consumer_groups).filled(:array?)
+
+      validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
+        # If there were no consumer_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        if consumer_groups.nil?
+          true
+        else
+          (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
+        end
+      end
+
+      validate(pid_existence: :pid) do |pid|
+        pid ? !File.exist?(pid) : true
+      end
+    end
+  end
+end

data/lib/karafka/server.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka consuming server class
+  class Server
+    @consumer_threads = Concurrent::Array.new
+
+    # How long should we sleep between checks on shutting down consumers
+    SUPERVISION_SLEEP = 1
+    # What system exit code should we use when we terminated forcefully
+    FORCEFUL_EXIT_CODE = 2
+
+    class << self
+      # Set of consuming threads. Each consumer thread contains a single consumer
+      attr_accessor :consumer_threads
+
+      # Writer for list of consumer groups that we want to consume in our current process context
+      attr_writer :consumer_groups
+
+      # Method which runs app
+      def run
+        process.on_sigint { stop_supervised }
+        process.on_sigquit { stop_supervised }
+        process.on_sigterm { stop_supervised }
+        start_supervised
+      end
+
+      # @return [Array<String>] array with names of consumer groups that should be consumed in a
+      #   current server context
+      def consumer_groups
+        # If not specified, a server will listed on all the topics
+        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
+      end
+
+      private
+
+      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
+      def process
+        Karafka::Process.instance
+      end
+
+      # Starts Karafka with a supervision
+      # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
+      # finish loop (and it won't happen until we explicitily want to stop)
+      def start_supervised
+        process.supervise
+        Karafka::App.run!
+        Karafka::Fetcher.call
+      end
+
+      # Stops Karafka with a supervision (as long as there is a shutdown timeout)
+      # If consumers won't stop in a given timeframe, it will force them to exit
+      def stop_supervised
+        # Because this is called in the trap context, there is a chance that instrumentation
+        # listeners contain things that aren't allowed from within a trap context.
+        # To bypass that (instead of telling users not to do things they need to)
+        # we spin up a thread to instrument server.stop and server.stop.error and wait until
+        # they're finished
+        Thread.new { Karafka.monitor.instrument('server.stop', {}) }.join
+
+        Karafka::App.stop!
+        # If there is no shutdown timeout, we don't exit and wait until all the consumers
+        # had done their work
+        return unless Karafka::App.config.shutdown_timeout
+
+        # If there is a timeout, we check every 1 second (for the timeout period) if all
+        # the threads finished their work and if so, we can just return and normal
+        # shutdown process will take place
+        Karafka::App.config.shutdown_timeout.to_i.times do
+          return if consumer_threads.count(&:alive?).zero?
+          sleep SUPERVISION_SLEEP
+        end
+
+        raise Errors::ForcefulShutdown
+      rescue Errors::ForcefulShutdown => error
+        Thread.new { Karafka.monitor.instrument('server.stop.error', error: error) }.join
+        # We're done waiting, lets kill them!
+        consumer_threads.each(&:terminate)
+
+        # exit is not within the instrumentation as it would not trigger due to exit
+        Kernel.exit FORCEFUL_EXIT_CODE
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module containing all Karafka setup related elements like configuration settings,
+  # config validations and configurators for external gems integration
+  module Setup
+    # Configurator for setting up all the framework details that are required to make it work
+    # @note If you want to do some configurations after all of this is done, please add to
+    #   karafka/config a proper file (needs to inherit from Karafka::Setup::Configurators::Base
+    #   and implement setup method) after that everything will happen automatically
+    # @note This config object allows to create a 1 level nestings (nodes) only. This should be
+    #   enough and will still keep the code simple
+    # @see Karafka::Setup::Configurators::Base for more details about configurators api
+    class Config
+      extend Dry::Configurable
+      extend Callbacks::Config
+
+      # Available settings
+      # option client_id [String] kafka client_id - used to provide
+      #   default Kafka groups namespaces and identify that app in kafka
+      setting :client_id
+      # What backend do we want to use to process messages
+      setting :backend, :inline
+      # option logger [Instance] logger that we want to use
+      setting :logger, -> { ::Karafka::Instrumentation::Logger.instance }
+      # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
+      setting :monitor, -> { ::Karafka::Instrumentation::Monitor.instance }
+      # Mapper used to remap consumer groups ids, so in case users migrate from other tools
+      # or they need to maintain their own internal consumer group naming conventions, they
+      # can easily do it, replacing the default client_id + consumer name pattern concept
+      setting :consumer_mapper, -> { Routing::ConsumerMapper }
+      # Mapper used to remap names of topics, so we can have a clean internal topic namings
+      # despite using any Kafka provider that uses namespacing, etc
+      # It needs to implement two methods:
+      #   - #incoming - for remapping from the incoming message to our internal format
+      #   - #outgoing - for remapping from internal topic name into outgoing message
+      setting :topic_mapper, -> { Routing::TopicMapper }
+      # Default parser for parsing and unparsing incoming and outgoing data
+      setting :parser, -> { Karafka::Parsers::Json }
+      # If batch_fetching is true, we will fetch kafka messages in batches instead of 1 by 1
+      # @note Fetching does not equal consuming, see batch_consuming description for details
+      setting :batch_fetching, true
+      # If batch_consuming is true, we will have access to #params_batch instead of #params.
+      # #params_batch will contain params received from Kafka (may be more than 1) so we can
+      # process them in batches
+      setting :batch_consuming, false
+      # Should we operate in a single consumer instance across multiple batches of messages,
+      # from the same partition or should we build a new one for each incoming batch.
+      # Disabling that can be useful when you want to create a new consumer instance for each
+      # incoming batch. It's disabled by default, not to create more objects that needed
+      # on each batch
+      setting :persistent, true
+      # option shutdown_timeout [Integer, nil] the number of seconds after which Karafka no
+      #   longer wait for the consumers to stop gracefully but instead we force
+      #   terminate everything.
+      # @note Keep in mind, that if your business logic
+      # @note If set to nil, it won't forcefully shutdown the process at all.
+      setting :shutdown_timeout, 60
+      # option params_base_class [Class] base class for params class initialization
+      #   This can be either a Hash or a HashWithIndifferentAccess depending on your
+      #   requirements. Note, that by using HashWithIndifferentAccess, you remove some of the
+      #   performance in favor of convenience. This can be useful especially if you already use
+      #   it with Rails, etc
+      setting :params_base_class, Hash
+
+      # option kafka [Hash] - optional - kafka configuration options
+      setting :kafka do
+        # Array with at least one host
+        setting :seed_brokers
+        # option session_timeout [Integer] the number of seconds after which, if a client
+        #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
+        setting :session_timeout, 30
+        # Time that a given partition will be paused from fetching messages, when message
+        # consumption fails. It allows us to process other partitions, while the error is being
+        # resolved and also "slows" things down, so it prevents from "eating" up all messages and
+        # consuming them with failed code. Use `nil` if you want to pause forever and never retry.
+        setting :pause_timeout, 10
+        # option offset_commit_interval [Integer] the interval between offset commits,
+        #   in seconds.
+        setting :offset_commit_interval, 10
+        # option offset_commit_threshold [Integer] the number of messages that can be
+        #   processed before their offsets are committed. If zero, offset commits are
+        #   not triggered by message consumption.
+        setting :offset_commit_threshold, 0
+        # option heartbeat_interval [Integer] the interval between heartbeats; must be less
+        #   than the session window.
+        setting :heartbeat_interval, 10
+        # option offset_retention_time [Integer] The length of the retention window, known as
+        #   offset retention time
+        setting :offset_retention_time, nil
+        # option fetcher_max_queue_size [Integer] max number of items in the fetch queue that
+        #   are stored for further processing. Note, that each item in the queue represents a
+        #   response from a single broker
+        setting :fetcher_max_queue_size, 100
+        # option max_bytes_per_partition [Integer] the maximum amount of data fetched
+        #   from a single partition at a time.
+        setting :max_bytes_per_partition, 1_048_576
+        #  whether to consume messages starting at the beginning or to just consume new messages
+        setting :start_from_beginning, true
+        # option min_bytes [Integer] the minimum number of bytes to read before
+        #   returning messages from the server; if `max_wait_time` is reached, this
+        #   is ignored.
+        setting :min_bytes, 1
+        # option max_bytes [Integer] the maximum number of bytes to read before returning messages
+        #   from each broker.
+        setting :max_bytes, 10_485_760
+        # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
+        #   wait before returning data from a single message fetch. By setting this high you also
+        #   increase the fetching throughput - and by setting it low you set a bound on latency.
+        #   This configuration overrides `min_bytes`, so you'll _always_ get data back within the
+        #   time specified. The default value is one second. If you want to have at most five
+        #   seconds of latency, set `max_wait_time` to 5. You should make sure
+        #   max_wait_time * num brokers + heartbeat_interval is less than session_timeout.
+        setting :max_wait_time, 1
+        # option automatically_mark_as_consumed [Boolean] should we automatically mark received
+        # messages as consumed (processed) after non-error consumption
+        setting :automatically_mark_as_consumed, true
+        # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
+        # Kafka cluster that went down (in seconds)
+        setting :reconnect_timeout, 5
+        # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
+        # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
+        # least one host.
+        setting :connect_timeout, 10
+        # option socket_timeout [Integer] Sets the number of seconds to wait when reading from or
+        # writing to a socket connection to a broker. After this timeout expires the connection
+        # will be killed. Note that some Kafka operations are by definition long-running, such as
+        # waiting for new messages to arrive in a partition, so don't set this value too low
+        setting :socket_timeout, 30
+
+        # SSL authentication related settings
+        # option ca_cert [String, nil] SSL CA certificate
+        setting :ssl_ca_cert, nil
+        # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
+        setting :ssl_ca_cert_file_path, nil
+        # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
+        #   certificate store
+        setting :ssl_ca_certs_from_system, false
+        # option ssl_client_cert [String, nil] SSL client certificate
+        setting :ssl_client_cert, nil
+        # option ssl_client_cert_key [String, nil] SSL client certificate password
+        setting :ssl_client_cert_key, nil
+        # option sasl_gssapi_principal [String, nil] sasl principal
+        setting :sasl_gssapi_principal, nil
+        # option sasl_gssapi_keytab [String, nil] sasl keytab
+        setting :sasl_gssapi_keytab, nil
+        # option sasl_plain_authzid [String] The authorization identity to use
+        setting :sasl_plain_authzid, ''
+        # option sasl_plain_username [String, nil] The username used to authenticate
+        setting :sasl_plain_username, nil
+        # option sasl_plain_password [String, nil] The password used to authenticate
+        setting :sasl_plain_password, nil
+        # option sasl_scram_username [String, nil] The username used to authenticate
+        setting :sasl_scram_username, nil
+        # option sasl_scram_password [String, nil] The password used to authenticate
+        setting :sasl_scram_password, nil
+        # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
+        setting :sasl_scram_mechanism, nil
+      end
+
+      class << self
+        # Configurating method
+        # @yield Runs a block of code providing a config singleton instance to it
+        # @yieldparam [Karafka::Setup::Config] Karafka config instance
+        def setup
+          configure { |config| yield(config) }
+        end
+
+        # Everything that should be initialized after the setup
+        # Components are in karafka/config directory and are all loaded one by one
+        # If you want to configure a next component, please add a proper file to config dir
+        def setup_components
+          [
+            Configurators::Params,
+            Configurators::WaterDrop
+          ].each { |klass| klass.setup(config) }
+        end
+
+        # Validate config based on ConfigurationSchema
+        # @return [Boolean] true if configuration is valid
+        # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
+        #   doesn't match with ConfigurationSchema
+        def validate!
+          validation_result = Karafka::Schemas::Config.call(config.to_h)
+
+          return true if validation_result.success?
+
+          raise Errors::InvalidConfiguration, validation_result.errors
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Configurators module is used to enclose all the external dependencies configurations
+    # upon which Karafka depents
+    class Configurators
+      # Karafka has some components that it relies on (like Sidekiq)
+      # We need to configure all of them only when the framework was set up.
+      # Any class that descends from this one will be automatically invoked upon setup (after it)
+      # @note This should be used only for internal Karafka dependencies configuration
+      #   End users configuration should go to the after_init block
+      # @example Configure an Example class
+      #   class ExampleConfigurator < Base
+      #     def setup
+      #       ExampleClass.logger = Karafka.logger
+      #       ExampleClass.redis = config.redis
+      #     end
+      #   end
+      class Base
+        # @param _config [Karafka::Config] config instance
+        # This method needs to be implemented in a subclass
+        def self.setup(_config)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/params.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    class Configurators
+      # Karafka::Params::Params are dynamically built based on user defined parent class
+      # so we cannot just require it, we need to initialize it after user is done with
+      # the framework configuration. This is a configurator that does exactly that.
+      class Params < Base
+        # Builds up Karafka::Params::Params class with user defined parent class
+        # @param config [Karafka::Setup::Config] Config we can user to setup things
+        def self.setup(config)
+          return if defined? Karafka::Params::Params
+
+          Karafka::Params.const_set(
+            'Params',
+            Class
+              .new(config.params_base_class)
+              .tap { |klass| klass.include(Karafka::Params::Dsl) }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/water_drop.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    class Configurators
+      # Class responsible for setting up WaterDrop configuration
+      class WaterDrop < Base
+        # Sets up a WaterDrop settings
+        # @param config [Karafka::Setup::Config] Config we can user to setup things
+        # @note This will also inject Karafka monitor as a default monitor into WaterDrop,
+        #   so we have the same monitor within whole Karafka framework (same with logger)
+        def self.setup(config)
+          ::WaterDrop.setup do |water_config|
+            water_config.deliver = true
+
+            config.to_h.reject { |k, _v| k == :kafka }.each do |k, v|
+              key_assignment = :"#{k}="
+              next unless water_config.respond_to?(key_assignment)
+              water_config.public_send(key_assignment, v)
+            end
+
+            config.kafka.to_h.each do |k, v|
+              key_assignment = :"#{k}="
+              next unless water_config.kafka.respond_to?(key_assignment)
+              water_config.kafka.public_send(key_assignment, v)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/dsl.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Dsl for allowing to work with the configuration from the Karafka::App
+    # @note Despite providing methods, everything is still persisted and fetched
+    # from the Karafka::Setup::Config
+    module Dsl
+      # Sets up the whole configuration
+      # @param [Block] block configuration block
+      def setup(&block)
+        Setup::Config.setup(&block)
+        initialize!
+      end
+
+      # @return [Karafka::Config] config instance
+      def config
+        Setup::Config.config
+      end
+    end
+  end
+end

data/lib/karafka/status.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  # App status monitor
+  class Status
+    include Singleton
+
+    # Available states and their transitions
+    STATES = {
+      initializing: :initialize!,
+      running: :run!,
+      stopped: :stop!
+    }.freeze
+
+    STATES.each do |state, transition|
+      define_method :"#{state}?" do
+        @status == state
+      end
+
+      define_method transition do
+        @status = state
+      end
+    end
+  end
+end