karafka 1.2.10 → 1.3.1

data/lib/karafka/instrumentation/listener.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    # Default listener that hooks up to our instrumentation and uses its events for logging
-    # It can be removed/replaced or anything without any harm to the Karafka app flow
-    module Listener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        debug
-        info
-        error
-        fatal
-      ].freeze
-
-      # Injects WaterDrop listener logger actions
-      extend WaterDrop::Instrumentation::Listener
-
-      class << self
-        # Logs details about incoming messages and with which consumer we will consume them
-        # @param event [Dry::Events::Event] event details including payload
-        def on_connection_delegator_call(event)
-          consumer = event[:consumer]
-          topic = consumer.topic.name
-          kafka_messages = event[:kafka_messages]
-          info "#{kafka_messages.count} messages on #{topic} topic delegated to #{consumer.class}"
-        end
-
-        # Logs details about each received message value parsing
-        # @param event [Dry::Events::Event] event details including payload
-        def on_params_params_parse(event)
-          # Keep in mind, that a caller here is a param object not a controller,
-          # so it returns a topic as a string, not a routing topic
-          debug "Params parsing for #{event[:caller].topic} topic successful in #{event[:time]} ms"
-        end
-
-        # Logs unsuccessful parsing attempts of incoming data
-        # @param event [Dry::Events::Event] event details including payload
-        def on_params_params_parse_error(event)
-          error "Params parsing error for #{event[:caller].topic} topic: #{event[:error]}"
-        end
-
-        # Logs errors that occured in a listener fetch loop
-        # @param event [Dry::Events::Event] event details including payload
-        # @note It's an error as we can recover from it not a fatal
-        def on_connection_listener_fetch_loop_error(event)
-          error "Listener fetch loop error: #{event[:error]}"
-        end
-
-        # Logs errors that are related to the connection itself
-        # @note Karafka will attempt to reconnect, so an error not a fatal
-        # @param event [Dry::Events::Event] event details including payload
-        def on_connection_client_fetch_loop_error(event)
-          error "Client fetch loop error: #{event[:error]}"
-        end
-
-        # Logs info about crashed fetcher
-        # @note If this happens, Karafka will shutdown as it means a critical error
-        #   in one of the threads
-        # @param event [Dry::Events::Event] event details including payload
-        def on_fetcher_call_error(event)
-          fatal "Fetcher crash due to an error: #{event[:error]}"
-        end
-
-        # Logs info about processing of a certain dataset with an inline backend
-        # @param event [Dry::Events::Event] event details including payload
-        def on_backends_inline_process(event)
-          count = event[:caller].send(:params_batch).to_a.size
-          topic = event[:caller].topic.name
-          time = event[:time]
-          info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
-        end
-
-        # Logs info about system signals that Karafka received
-        # @param event [Dry::Events::Event] event details including payload
-        def on_process_notice_signal(event)
-          info "Received #{event[:signal]} system signal"
-        end
-
-        # Logs info about responder usage withing a controller flow
-        # @param event [Dry::Events::Event] event details including payload
-        def on_consumers_responders_respond_with(event)
-          calling = event[:caller].class
-          responder = calling.topic.responder
-          data = event[:data]
-          info "Responded from #{calling} using #{responder} with following data #{data}"
-        end
-
-        # Logs info that we're going to stop the Karafka server
-        # @param _event [Dry::Events::Event] event details including payload
-        def on_server_stop(_event)
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { info "Stopping Karafka server #{::Process.pid}" }
-        end
-
-        # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
-        #   forced exit
-        # @param _event [Dry::Events::Event] event details including payload
-        def on_server_stop_error(_event)
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
-        end
-
-        USED_LOG_LEVELS.each do |log_level|
-          define_method log_level do |*args|
-            Karafka.logger.send(log_level, *args)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/loader.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Loader for requiring all the files in a proper order
-  module Loader
-    # Order in which we want to load app files
-    DIRS = %w[
-      lib
-      app
-    ].freeze
-
-    # Will load files in a proper order (based on DIRS)
-    # @param [String] root path from which we want to start
-    def self.load(root)
-      DIRS.each do |dir|
-        path = File.join(root, dir)
-        next unless File.exist?(path)
-        load!(path)
-      end
-    end
-
-    # Requires all the ruby files from one path in a proper order
-    # @param path [String] path (dir) from which we want to load ruby files in a proper order
-    def self.load!(path)
-      require_all(path)
-    end
-  end
-end

data/lib/karafka/params/dsl.rb

@@ -1,158 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Params namespace encapsulating all the logic that is directly related to params handling
-  module Params
-    # Dsl for Karafka params. We don't provide the params class here as we want to allow users to
-    # use either hash (default) or Rails hash with indifferent access as a base for their params
-    #
-    # We do that because both of them have their own advantages and we don't want to enforce users
-    # to handle things differently if they already use any of those
-    #
-    # It provides lazy loading not only until the first usage, but also allows us to skip
-    # using parser until we execute our logic. That way we can operate with
-    # heavy-parsing data without slowing down the whole application.
-    module Dsl
-      # Params keys that are "our" and internal. We use this list for additional backends
-      # that somehow operatae on those keys
-      SYSTEM_KEYS = %w[
-        parser
-        value
-        partition
-        offset
-        key
-        create_time
-        receive_time
-        topic
-        parsed
-      ].freeze
-
-      # Params attributes that should be available via a method call invocation for Kafka
-      # client compatibility.
-      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
-      # uses those fields via method calls, so in order to be able to pass there our params
-      # objects, have to have same api.
-      METHOD_ATTRIBUTES = %w[
-        topic
-        partition
-        offset
-        key
-        create_time
-        receive_time
-      ].freeze
-
-      private_constant :METHOD_ATTRIBUTES
-
-      # Class methods required by params to work
-      module ClassMethods
-        # We allow building instances only via the #build method
-
-        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
-        #   in case of building params inside main Karafka process in
-        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
-        # @param parser [Class] parser class that we will use to unparse data
-        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
-        #   retrieving data that we've got from Kafka
-        # @example Build params instance from a hash
-        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
-        # @example Build params instance from a Kafka::FetchedMessage object
-        #   Karafka::Params::Params.build(message) #=> params object
-        def build(message, parser)
-          instance = new
-          instance['parser'] = parser
-
-          # Non kafka fetched message can happen when we interchange data with an
-          # additional backend
-          if message.is_a?(Kafka::FetchedMessage)
-            instance.send(
-              :merge!,
-              'value' => message.value,
-              'partition' => message.partition,
-              'offset' => message.offset,
-              'key' => message.key,
-              'create_time' => message.create_time,
-              'receive_time' => Time.now,
-              # When we get raw messages, they might have a topic, that was modified by a
-              # topic mapper. We need to "reverse" this change and map back to the non-modified
-              # format, so our internal flow is not corrupted with the mapping
-              'topic' => Karafka::App.config.topic_mapper.incoming(message.topic)
-            )
-          else
-            instance.send(:merge!, message)
-          end
-
-          instance
-        end
-      end
-
-      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
-      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
-      #   to the current object. This object will be also marked as already parsed, so we won't
-      #   parse it again.
-      def retrieve!
-        return self if self['parsed']
-        self['parsed'] = true
-
-        parsed_data = parse(self['value'])
-        delete('value')
-        merge!(parsed_data)
-      end
-
-      # Includes and extends the base params klass with everything that is needed by Karafka to
-      #   fully work in any conditions.
-      # @param params_klass [Karafka::Params::Params] initialized params class that we will
-      #   use for a given Karafka process
-      def self.included(params_klass)
-        params_klass.extend(Dsl::ClassMethods)
-
-        METHOD_ATTRIBUTES.each do |attr|
-          # Defines a method call accessor to a particular hash field.
-          # @note Won't work for complex key names that contain spaces, etc
-          # @param key [Symbol] name of a field that we want to retrieve with a method call
-          # @example
-          #   key_attr_reader :example
-          #   params.example #=> 'my example value'
-          params_klass.send :define_method, attr do
-            self[attr]
-          end
-        end
-
-        params_klass.send :private, :merge!
-        params_klass.send :private, :parse
-      end
-
-      private
-
-      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
-      #  and in a other_hash - it will not replace keys that are the same in our hash
-      #  and in the other one. This protects some important Karafka params keys that cannot be
-      #  replaced with custom values from incoming Kafka message
-      # @param other_hash [Hash] hash that we want to merge into current
-      # @return [Karafka::Params::Params] our parameters hash with merged values
-      # @example Merge with hash without same keys
-      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
-      # @example Merge with hash with same keys (symbol based)
-      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
-      # @example Merge with hash with same keys (string based)
-      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
-      # @example Merge with hash with same keys (current string based)
-      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
-      def merge!(other_hash)
-        super(other_hash) { |_key, base_value, _new_value| base_value }
-      end
-
-      # @param value [String] Raw data that we want to parse using consumer parser
-      # @note If something goes wrong, it will return raw data in a hash with a message key
-      # @return [Hash] parsed data or a hash with message key containing raw data if something
-      #   went wrong during parsing
-      def parse(value)
-        Karafka.monitor.instrument('params.params.parse', caller: self) do
-          self['parser'].parse(value)
-        end
-      rescue ::Karafka::Errors::ParserError => e
-        Karafka.monitor.instrument('params.params.parse.error', caller: self, error: e)
-        raise e
-      end
-    end
-  end
-end

data/lib/karafka/parsers/json.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module for all supported by default parsers for incoming/outgoing data
-  module Parsers
-    # Default Karafka Json parser for serializing and deserializing data
-    class Json
-      # @param content [String] content based on which we want to get our hash
-      # @return [Hash] hash with parsed JSON data
-      # @example
-      #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
-      def self.parse(content)
-        ::MultiJson.load(content)
-      rescue ::MultiJson::ParseError => e
-        raise ::Karafka::Errors::ParserError, e
-      end
-
-      # @param content [Object] any object that we want to convert to a json string
-      # @return [String] Valid JSON string containing serialized data
-      # @raise [Karafka::Errors::ParserError] raised when we don't have a way to parse
-      #   given content to a json string format
-      # @note When string is passed to this method, we assume that it is already a json
-      #   string and we don't serialize it again. This allows us to serialize data before
-      #   it is being forwarded to a parser if we want to have a custom (not that simple)
-      #   json serialization
-      #
-      # @example From an ActiveRecord object
-      #   Json.generate(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
-      # @example From a string (no changes)
-      #   Json.generate("{\"a\":1}") #=> "{\"a\":1}"
-      def self.generate(content)
-        return content if content.is_a?(String)
-        return content.to_json if content.respond_to?(:to_json)
-        raise Karafka::Errors::ParserError, content
-      end
-    end
-  end
-end

data/lib/karafka/patches/dry_configurable.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for patches of external gems/libraries
-  module Patches
-    # Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
-    # @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
-    module DryConfigurable
-      # We overwrite ::Dry::Configurable::Config to change on proc behaviour
-      # Unfortunately it does not provide an on call proc evaluation, so
-      # this feature had to be added here on demand/
-      # @param args Any arguments that DryConfigurable::Config accepts
-      def initialize(*args)
-        super
-
-        @config.each_key(&method(:rebuild))
-      end
-
-      private
-
-      # Method that rebuilds a given accessor, so when it consists a proc value, it will
-      # evaluate it upon return for blocks that don't require any arguments, otherwise
-      # it will return the block
-      # @param method_name [Symbol] name of an accessor that we want to rebuild
-      def rebuild(method_name)
-        define_singleton_method method_name do
-          value = super()
-          return value unless value.is_a?(Proc)
-          return value unless value.parameters.empty?
-          value.call
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/topic.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Local cache for routing topics
-    # We use it in order not to build string instances and remap incoming topic upon each
-    # message / message batches received
-    class Topic
-      # Thread.current scope under which we store topics data
-      PERSISTENCE_SCOPE = :topics
-
-      # @param group_id [String] group id for which we fetch a topic representation
-      # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
-      #   topic representation
-      # @return [Karafka::Routing::Topic] remapped topic representation that can be used further
-      #   on when working with given parameters
-      def self.fetch(group_id, raw_topic_name)
-        Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
-
-        Thread.current[PERSISTENCE_SCOPE][group_id][raw_topic_name] ||= begin
-          # We map from incoming topic name, as it might be namespaced, etc.
-          # @see topic_mapper internal docs
-          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
-          Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-        end
-      end
-    end
-  end
-end

data/lib/karafka/schemas/config.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all the validation schemas that we use to check input
-  module Schemas
-    # Regexp for validating format of groups and topics
-    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
-
-    # Schema with validation rules for Karafka configuration details
-    # @note There are many more configuration options inside of the
-    #   Karafka::Setup::Config model, but we don't validate them here as they are
-    #   validated per each route (topic + consumer_group) because they can be overwritten,
-    #   so we validate all of that once all the routes are defined and ready
-    Config = Dry::Validation.Schema do
-      required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:shutdown_timeout) { none? | (int? & gteq?(0)) }
-      required(:consumer_mapper)
-      required(:topic_mapper)
-      required(:params_base_class).filled
-
-      optional(:backend).filled
-    end
-  end
-end

data/lib/karafka/schemas/consumer_group.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Schema for single full route (consumer group + topics) validation.
-    ConsumerGroup = Dry::Validation.Schema do
-      # Valid uri schemas of Kafka broker url
-      # The ||= is due to the behavior of require_all that resolves dependencies
-      # but someetimes loads things twice
-      URI_SCHEMES ||= %w[kafka kafka+ssl plaintext ssl].freeze
-
-      # Available sasl scram mechanism of authentication (plus nil)
-      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
-
-      configure do
-        config.messages_file = File.join(
-          Karafka.gem_root, 'config', 'errors.yml'
-        )
-
-        # Uri validator to check if uri is in a Karafka acceptable format
-        # @param uri [String] uri we want to validate
-        # @return [Boolean] true if it is a valid uri, otherwise false
-        def broker_schema?(uri)
-          uri = URI.parse(uri)
-          URI_SCHEMES.include?(uri.scheme) && uri.port
-        rescue URI::InvalidURIError
-          false
-        end
-      end
-
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:seed_brokers).filled { each(:broker_schema?) }
-      required(:session_timeout).filled { int? | float? }
-      required(:pause_timeout) { none? | ((int? | float?) & gteq?(0)) }
-      required(:offset_commit_interval) { int? | float? }
-      required(:offset_commit_threshold).filled(:int?)
-      required(:offset_retention_time) { none?.not > int? }
-      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
-      required(:fetcher_max_queue_size).filled(:int?, gt?: 0)
-      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
-      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
-      required(:min_bytes).filled(:int?, gt?: 0)
-      required(:max_bytes).filled(:int?, gt?: 0)
-      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
-      required(:batch_fetching).filled(:bool?)
-      required(:topics).filled { each { schema(ConsumerGroupTopic) } }
-
-      # Max wait time cannot exceed socket_timeout - wouldn't make sense
-      rule(
-        max_wait_time_limit: %i[max_wait_time socket_timeout]
-      ) do |max_wait_time, socket_timeout|
-        socket_timeout.int? > max_wait_time.lteq?(value(:socket_timeout))
-      end
-
-      %i[
-        ssl_ca_cert
-        ssl_ca_cert_file_path
-        ssl_client_cert
-        ssl_client_cert_key
-        sasl_gssapi_principal
-        sasl_gssapi_keytab
-        sasl_plain_authzid
-        sasl_plain_username
-        sasl_plain_password
-        sasl_scram_username
-        sasl_scram_password
-      ].each do |encryption_attribute|
-        optional(encryption_attribute).maybe(:str?)
-      end
-
-      optional(:ssl_ca_certs_from_system).maybe(:bool?)
-
-      # It's not with other encryptions as it has some more rules
-      optional(:sasl_scram_mechanism)
-        .maybe(:str?, included_in?: Karafka::Schemas::SASL_SCRAM_MECHANISMS)
-    end
-  end
-end