karafka 1.1.0 → 1.3.0

data/lib/karafka/monitor.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Monitor is used to hookup external monitoring services to monitor how Karafka works
-  # It provides a standarized API for checking incoming messages/enqueueing etc
-  # By default it implements logging functionalities but can be replaced with any more
-  #   sophisticated logging/monitoring system like Errbit, Airbrake, NewRelic
-  # @note This class acts as a singleton because we are only permitted to have single monitor
-  #   per running process (just as logger)
-  # Keep in mind, that if you create your own monitor object, you will have to implement also
-  # logging functionality (or just inherit, super and do whatever you want)
-  class Monitor
-    include Singleton
-
-    # This method is executed in many important places in the code (during data flow), like
-    # the moment before #consume_async, etc. For full list just grep for 'monitor.notice'
-    # @param caller_class [Class] class of object that executed this call
-    # @param options [Hash] hash with options that we passed to notice. It differs based
-    #   on of who and when is calling
-    # @note We don't provide a name of method in which this was called, because we can take
-    #   it directly from Ruby (see #caller_label method of this class for more details)
-    # @example Notice about consuming with controller_class
-    #   Karafka.monitor.notice(self.class, controller_class: controller_class)
-    # @example Notice about terminating with a signal
-    #   Karafka.monitor.notice(self.class, signal: signal)
-    def notice(caller_class, options = {})
-      logger.info("#{caller_class}##{caller_label} with #{options}")
-    end
-
-    # This method is executed when we want to notify about an error that happened somewhere
-    # in the system
-    # @param caller_class [Class] class of object that executed this call
-    # @param e [Exception] exception that was raised
-    # @note We don't provide a name of method in which this was called, because we can take
-    #   it directly from Ruby (see #caller_label method of this class for more details)
-    # @example Notify about error
-    #   Karafka.monitor.notice(self.class, e)
-    def notice_error(caller_class, e)
-      caller_exceptions_map.each do |level, types|
-        next unless types.include?(caller_class)
-
-        return logger.public_send(level, e)
-      end
-
-      logger.info(e)
-    end
-
-    private
-
-    # @return [Hash] Hash containing informations on which level of notification should
-    #   we use for exceptions that happen in certain parts of Karafka
-    # @note Keep in mind that any not handled here class should be logged with info
-    # @note Those are not maps of exceptions classes but of classes that were callers of this
-    #   particular exception
-    def caller_exceptions_map
-      @caller_exceptions_map ||= {
-        error: [
-          Karafka::Connection::Consumer,
-          Karafka::Connection::Listener,
-          Karafka::Params::Params
-        ],
-        fatal: [
-          Karafka::Fetcher
-        ]
-      }
-    end
-
-    # @return [String] label of method that invoked #notice or #notice_error
-    # @example Check label of method that invoked #notice
-    #   caller_label #=> 'fetch'
-    # @example Check label of method that invoked #notice in a block
-    #   caller_label #=> 'block in fetch'
-    # @example Check label of method that invoked #notice_error
-    #   caller_label #=> 'rescue in target'
-    def caller_label
-      # We need to calculate ancestors because if someone inherits
-      # from this  class, caller chains is longer
-      index = self.class.ancestors.index(Karafka::Monitor)
-      # caller_locations has a differs in result whether it is a subclass of
-      # Karafka::Monitor, the basic Karafka::Monitor itself or a super for a subclass.
-      # So to cover all the cases we need to differentiate.
-      # @see https://github.com/karafka/karafka/issues/128
-      # @note It won't work if the monitor caller_label caller class is defined using
-      #   define method
-      super_execution = caller_locations(1, 2)[0].label == caller_locations(1, 2)[1].label
-
-      scope = super_execution ? 1 : nil
-      scope ||= index.positive? ? 0 : 1
-
-      caller_locations(index + 1, 2)[scope].label
-    end
-
-    # @return [Logger] logger instance
-    def logger
-      Karafka.logger
-    end
-  end
-end

data/lib/karafka/parsers/json.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module for all supported by default parsers for incoming/outgoing data
-  module Parsers
-    # Default Karafka Json parser for serializing and deserializing data
-    class Json
-      # @param content [String] content based on which we want to get our hash
-      # @return [Hash] hash with parsed JSON data
-      # @example
-      #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
-      def self.parse(content)
-        ::MultiJson.load(content)
-      rescue ::MultiJson::ParseError => e
-        raise ::Karafka::Errors::ParserError, e
-      end
-
-      # @param content [Object] any object that we want to convert to a json string
-      # @return [String] Valid JSON string containing serialized data
-      # @raise [Karafka::Errors::ParserError] raised when we don't have a way to parse
-      #   given content to a json string format
-      # @note When string is passed to this method, we assume that it is already a json
-      #   string and we don't serialize it again. This allows us to serialize data before
-      #   it is being forwarded to a parser if we want to have a custom (not that simple)
-      #   json serialization
-      #
-      # @example From an ActiveRecord object
-      #   Json.generate(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
-      # @example From a string (no changes)
-      #   Json.generate("{\"a\":1}") #=> "{\"a\":1}"
-      def self.generate(content)
-        return content if content.is_a?(String)
-        return content.to_json if content.respond_to?(:to_json)
-        raise Karafka::Errors::ParserError, content
-      end
-    end
-  end
-end

data/lib/karafka/patches/dry_configurable.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for patches of external gems/libraries
-  module Patches
-    # Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
-    # @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
-    module DryConfigurable
-      # We overwrite ::Dry::Configurable::Config to change on proc behaviour
-      # Unfortunately it does not provide an on call proc evaluation, so
-      # this feature had to be added here on demand/
-      # @param args Any arguments that DryConfigurable::Config accepts
-      def initialize(*args)
-        super
-
-        @config.each_key(&method(:rebuild))
-      end
-
-      private
-
-      # Method that rebuilds a given accessor, so when it consists a proc value, it will
-      # evaluate it upon return
-      # @param method_name [Symbol] name of an accessor that we want to rebuild
-      def rebuild(method_name)
-        define_singleton_method method_name do
-          super().is_a?(Proc) ? super().call : super()
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/consumer.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Persistence layer to store current thread messages consumer for further use
-    class Consumer
-      # Thread.current key under which we store current thread messages consumer
-      PERSISTENCE_SCOPE = :consumer
-
-      # @param consumer [Karafka::Connection::Consumer] messages consumer of
-      #   a current thread
-      # @return [Karafka::Connection::Consumer] persisted messages consumer
-      def self.write(consumer)
-        Thread.current[PERSISTENCE_SCOPE] = consumer
-      end
-
-      # @return [Karafka::Connection::Consumer] persisted messages consumer
-      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
-      #   but we try to use it anyway
-      def self.read
-        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingConsumer)
-      end
-    end
-  end
-end

data/lib/karafka/persistence/controller.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module used to provide a persistent cache layer for Karafka components that need to be
-  # shared inside of a same thread
-  module Persistence
-    # Module used to provide a persistent cache across batch requests for a given
-    # topic and partition to store some additional details when the persistent mode
-    # for a given topic is turned on
-    class Controller
-      # Thread.current scope under which we store controllers data
-      PERSISTENCE_SCOPE = :controllers
-
-      class << self
-        # @return [Hash] current thread persistence scope hash with all the controllers
-        def all
-          Thread.current[PERSISTENCE_SCOPE] ||= {}
-        end
-
-        # Used to build (if block given) and/or fetch a current controller instance that will be
-        #   used to process messages from a given topic and partition
-        # @return [Karafka::BaseController] base controller descendant
-        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
-        # @param partition [Integer] number of partition for which we want to cache
-        def fetch(topic, partition)
-          all[topic.id] ||= {}
-
-          # We always store a current instance
-          if topic.persistent
-            all[topic.id][partition] ||= yield
-          else
-            all[topic.id][partition] = yield
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/schemas/config.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all the validation schemas that we use to check input
-  module Schemas
-    # Regexp for validating format of groups and topics
-    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
-
-    # Schema with validation rules for Karafka configuration details
-    # @note There are many more configuration options inside of the
-    #   Karafka::Setup::Config model, but we don't validate them here as they are
-    #   validated per each route (topic + consumer_group) because they can be overwritten,
-    #   so we validate all of that once all the routes are defined and ready
-    Config = Dry::Validation.Schema do
-      required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:consumer_mapper)
-      required(:topic_mapper)
-      optional(:backend).filled
-    end
-  end
-end

data/lib/karafka/schemas/consumer_group.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Schema for single full route (consumer group + topics) validation.
-    ConsumerGroup = Dry::Validation.Schema do
-      # Valid uri schemas of Kafka broker url
-      # The ||= is due to the behavior of require_all that resolves dependencies
-      # but someetimes loads things twice
-      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
-
-      configure do
-        config.messages_file = File.join(
-          Karafka.gem_root, 'config', 'errors.yml'
-        )
-
-        # Uri validator to check if uri is in a Karafka acceptable format
-        # @param uri [String] uri we want to validate
-        # @return [Boolean] true if it is a valid uri, otherwise false
-        def broker_schema?(uri)
-          uri = URI.parse(uri)
-          URI_SCHEMES.include?(uri.scheme) && uri.port
-        rescue URI::InvalidURIError
-          return false
-        end
-      end
-
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:seed_brokers).filled { each(:broker_schema?) }
-      required(:session_timeout).filled { int? | float? }
-      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
-      required(:offset_commit_interval) { int? | float? }
-      required(:offset_commit_threshold).filled(:int?)
-      required(:offset_retention_time) { none?.not > int? }
-      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
-      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
-      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
-      required(:min_bytes).filled(:int?, gt?: 0)
-      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
-      required(:batch_fetching).filled(:bool?)
-      required(:topics).filled { each { schema(ConsumerGroupTopic) } }
-
-      # Max wait time cannot exceed socket_timeout - wouldn't make sense
-      rule(
-        max_wait_time_limit: %i[max_wait_time socket_timeout]
-      ) do |max_wait_time, socket_timeout|
-        socket_timeout.int? > max_wait_time.lteq?(value(:socket_timeout))
-      end
-
-      %i[
-        ssl_ca_cert
-        ssl_ca_cert_file_path
-        ssl_client_cert
-        ssl_client_cert_key
-        sasl_plain_authzid
-        sasl_plain_username
-        sasl_plain_password
-        sasl_gssapi_principal
-        sasl_gssapi_keytab
-      ].each do |encryption_attribute|
-        optional(encryption_attribute).maybe(:str?)
-      end
-    end
-  end
-end

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Consumer group topic validation rules
-    ConsumerGroupTopic = Dry::Validation.Schema do
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:backend).filled(included_in?: %i[inline sidekiq])
-      required(:controller).filled
-      required(:parser).filled
-      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
-      required(:start_from_beginning).filled(:bool?)
-      required(:batch_consuming).filled(:bool?)
-      required(:persistent).filled(:bool?)
-    end
-  end
-end

data/lib/karafka/schemas/responder_usage.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Validator to check responder topic usage
-    ResponderUsageTopic = Dry::Validation.Schema do
-      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:required).filled(:bool?)
-      required(:multiple_usage).filled(:bool?)
-      required(:usage_count).filled(:int?, gteq?: 0)
-      required(:registered).filled(eql?: true)
-      required(:async).filled(:bool?)
-
-      rule(
-        required_usage: %i[required usage_count]
-      ) do |required, usage_count|
-        required.true? > usage_count.gteq?(1)
-      end
-
-      rule(
-        multiple_usage_permission: %i[multiple_usage usage_count]
-      ) do |multiple_usage, usage_count|
-        usage_count.gt?(1) > multiple_usage.true?
-      end
-
-      rule(
-        multiple_usage_block: %i[multiple_usage usage_count]
-      ) do |multiple_usage, usage_count|
-        multiple_usage.false? > usage_count.lteq?(1)
-      end
-    end
-
-    # Validator to check that everything in a responder flow matches responder rules
-    ResponderUsage = Dry::Validation.Schema do
-      required(:used_topics) { filled? > each { schema(ResponderUsageTopic) } }
-      required(:registered_topics) { filled? > each { schema(ResponderUsageTopic) } }
-    end
-  end
-end

data/lib/karafka/schemas/server_cli_options.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Schemas
-    # Schema for validating correctness of the server cli command options
-    # We validate some basics + the list of consumer_groups on which we want to use, to make
-    # sure that all of them are defined, plus that a pidfile does not exist
-    ServerCliOptions = Dry::Validation.Schema do
-      configure do
-        option :consumer_groups
-
-        def self.messages
-          super.merge(
-            en: {
-              errors: {
-                consumer_groups_inclusion: 'Unknown consumer group.',
-                pid_existence: 'Pidfile already exists.'
-              }
-            }
-          )
-        end
-      end
-
-      optional(:pid).filled(:str?)
-      optional(:daemon).filled(:bool?)
-      optional(:consumer_groups).filled(:array?)
-
-      validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
-        # If there were no consumer_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        if consumer_groups.nil?
-          true
-        else
-          (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
-        end
-      end
-
-      validate(pid_existence: :pid) do |pid|
-        pid ? !File.exist?(pid) : true
-      end
-    end
-  end
-end

data/lib/karafka/setup/configurators/base.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Setup
-    # Configurators module is used to enclose all the external dependencies configurations
-    class Configurators
-      # Karafka has come components that it relies on (like Sidekiq)
-      # We need to configure all of them only when the framework was set up.
-      # Any class that descends from this one will be automatically invoked upon setup (after it)
-      # @example Configure an Example class
-      #   class ExampleConfigurator < Base
-      #     def setup
-      #       ExampleClass.logger = Karafka.logger
-      #       ExampleClass.redis = config.redis
-      #     end
-      #   end
-      class Base
-        extend ActiveSupport::DescendantsTracker
-
-        attr_reader :config
-
-        # @param config [Karafka::Config] config instance
-        # @return [Karafka::Config::Base] configurator for a given component
-        def initialize(config)
-          @config = config
-        end
-
-        # This method needs to be implemented in a subclass
-        def setup
-          raise NotImplementedError
-        end
-      end
-    end
-  end
-end