karafka 1.4.0.rc1

data/lib/karafka/contracts/responder_usage.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Validator to check responder topic usage
+    class ResponderUsageTopic < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      params do
+        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:required).filled(:bool?)
+        required(:usage_count).filled(:int?, gteq?: 0)
+        required(:registered).filled(eql?: true)
+        required(:async).filled(:bool?)
+        required(:serializer).filled
+      end
+
+      rule(:required, :usage_count) do
+        key(:name).failure(:required_usage_count) if values[:required] && values[:usage_count] < 1
+      end
+    end
+
+    # Validator to check that everything in a responder flow matches responder rules
+    class ResponderUsage < Dry::Validation::Contract
+      include Dry::Core::Constants
+
+      # Contract for verifying the topic usage details
+      TOPIC_CONTRACT = ResponderUsageTopic.new.freeze
+
+      private_constant :TOPIC_CONTRACT
+
+      params do
+        required(:used_topics)
+        required(:registered_topics)
+      end
+
+      rule(:used_topics) do
+        (value || EMPTY_ARRAY).each do |used_topic|
+          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
+            key([:used_topics, used_topic, error.path[0]]).failure(error.text)
+          end
+        end
+      end
+
+      rule(:registered_topics) do
+        (value || EMPTY_ARRAY).each do |used_topic|
+          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
+            key([:registered_topics, used_topic, error.path[0]]).failure(error.text)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/contracts/server_cli_options.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract for validating correctness of the server cli command options
+    # We validate some basics + the list of consumer_groups on which we want to use, to make
+    # sure that all of them are defined, plus that a pidfile does not exist
+    class ServerCliOptions < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      params do
+        optional(:pid).filled(:str?)
+        optional(:daemon).filled(:bool?)
+        optional(:consumer_groups).value(:array, :filled?)
+      end
+
+      rule(:pid) do
+        key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
+      end
+
+      rule(:consumer_groups) do
+        # If there were no consumer_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        if !value.nil? &&
+           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
+          key(:consumer_groups).failure(:consumer_groups_inclusion)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace used to encapsulate all the internal errors of Karafka
+  module Errors
+    # Base class for all the Karafka internal errors
+    BaseError = Class.new(StandardError)
+
+    # Should be raised when we have that that we cannot serialize
+    SerializationError = Class.new(BaseError)
+
+    # Should be raised when we tried to deserialize incoming data but we failed
+    DeserializationError = Class.new(BaseError)
+
+    # Raised when router receives topic name which does not correspond with any routes
+    # This can only happen in a case when:
+    #   - you've received a message and we cannot match it with a consumer
+    #   - you've changed the routing, so router can no longer associate your topic to
+    #     any consumer
+    #   - or in a case when you do a lot of meta-programming and you change routing/etc on runtime
+    #
+    # In case this happens, you will have to create a temporary route that will allow
+    # you to "eat" everything from the Sidekiq queue.
+    # @see https://github.com/karafka/karafka/issues/135
+    NonMatchingRouteError = Class.new(BaseError)
+
+    # Raised when we don't use or use responder not in the way it expected to based on the
+    # topics usage definitions
+    InvalidResponderUsageError = Class.new(BaseError)
+
+    # Raised when options that we provide to the responder to respond aren't what the contract
+    # requires
+    InvalidResponderMessageOptionsError = Class.new(BaseError)
+
+    # Raised when configuration doesn't match with validation contract
+    InvalidConfigurationError = Class.new(BaseError)
+
+    # Raised when we try to use Karafka CLI commands (except install) without a boot file
+    MissingBootFileError = Class.new(BaseError)
+
+    # Raised when we want to read a persisted thread messages consumer but it is unavailable
+    # This should never happen and if it does, please contact us
+    MissingClientError = Class.new(BaseError)
+
+    # Raised when want to hook up to an event that is not registered and supported
+    UnregisteredMonitorEventError = Class.new(BaseError)
+
+    # Raised when we've waited enough for shutting down a non-responsive process
+    ForcefulShutdownError = Class.new(BaseError)
+  end
+end

data/lib/karafka/fetcher.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to run the Karafka consumer and handle shutting down, restarting etc
+  # @note Creating multiple fetchers will result in having multiple connections to the same
+  #   topics, which means that if there are no partitions, it won't use them.
+  class Fetcher
+    # Starts listening on all the listeners asynchronously
+    # Fetch loop should never end, which means that we won't create more actor clusters
+    # so we don't have to terminate them
+    def call
+      threads = listeners.map do |listener|
+        # We abort on exception because there should be an exception handling developed for
+        # each listener running in separate threads, so the exceptions should never leak
+        # and if that happens, it means that something really bad happened and we should stop
+        # the whole process
+        Thread
+          .new { listener.call }
+          .tap { |thread| thread.abort_on_exception = true }
+      end
+
+      # We aggregate threads here for a supervised shutdown process
+      threads.each { |thread| Karafka::Server.consumer_threads << thread }
+      threads.each(&:join)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something terrible happened
+    rescue StandardError => e
+      Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
+      Karafka::App.stop!
+      raise e
+    end
+
+    private
+
+    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+    def listeners
+      @listeners ||= App.consumer_groups.active.map do |consumer_group|
+        Karafka::Connection::Listener.new(consumer_group)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/class_matcher.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Class used to autodetect corresponding classes that are internally inside Karafka framework
+    # It is used among others to match:
+    #   consumer => responder
+    class ClassMatcher
+      # Regexp used to remove any non classy like characters that might be in the consumer
+      # class name (if defined dynamically, etc)
+      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}.freeze
+
+      private_constant :CONSTANT_REGEXP
+
+      # @param klass [Class] class to which we want to find a corresponding class
+      # @param from [String] what type of object is it (based on postfix name part)
+      # @param to [String] what are we looking for (based on a postfix name part)
+      # @example Consumer that has a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperConsumer, 'Consumer', 'Responder')
+      #   matcher.match #=> SuperResponder
+      # @example Consumer without a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Consumer, 'Consumer', 'Responder')
+      #   matcher.match #=> nil
+      def initialize(klass, from:, to:)
+        @klass = klass
+        @from = from
+        @to = to
+      end
+
+      # @return [Class] matched class
+      # @return [nil] nil if we couldn't find matching class
+      def match
+        return nil if name.empty?
+        return nil unless scope.const_defined?(name)
+
+        matching = scope.const_get(name)
+        same_scope?(matching) ? matching : nil
+      end
+
+      # @return [String] name of a new class that we're looking for
+      # @note This method returns name of a class without a namespace
+      # @example From SuperConsumer matching responder
+      #   matcher.name #=> 'SuperResponder'
+      # @example From Namespaced::Super2Consumer matching responder
+      #   matcher.name #=> Super2Responder
+      def name
+        inflected = +@klass.to_s.split('::').last.to_s
+        # We inject the from into the name just in case it is missing as in a situation like
+        # that it would just sanitize the name without adding the "to" postfix.
+        # It could create cases when we want to build for example a responder to a consumer
+        # that does not have the "Consumer" postfix and would do nothing returning the same name.
+        # That would be bad as the matching classes shouldn't be matched to themselves.
+        inflected << @from unless inflected.include?(@from)
+        inflected.gsub!(@from, @to)
+        inflected.gsub!(CONSTANT_REGEXP, '')
+        inflected
+      end
+
+      # @return [Class, Module] class or module in which we're looking for a matching
+      def scope
+        scope_of(@klass)
+      end
+
+      private
+
+      # @param klass [Class] class for which we want to extract it's enclosing class/module
+      # @return [Class, Module] enclosing class/module
+      # @return [::Object] object if it was a root class
+      #
+      # @example Non-namespaced class
+      #   scope_of(SuperClass) #=> Object
+      # @example Namespaced class
+      #   scope_of(Abc::SuperClass) #=> Abc
+      def scope_of(klass)
+        enclosing = klass.to_s.split('::')[0...-1]
+        return ::Object if enclosing.empty?
+
+        ::Object.const_get(enclosing.join('::'))
+      end
+
+      # @param matching [Class] class of which scope we want to check
+      # @return [Boolean] true if the scope of class is the same as scope of matching
+      def same_scope?(matching)
+        scope == scope_of(matching)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/config_retriever.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # A helper method that allows us to build methods that try to get a given
+    # attribute from its instance value and if it fails, will fallback to
+    # the default config or config.kafka value for a given attribute.
+    # It is used to simplify the checks.
+    # @note Worth noticing, that the value might be equal to false, so even
+    #   then we need to return it. That's why we check for nil?
+    # @example Define config retried attribute for start_from_beginning
+    # class Test
+    #   extend Karafka::Helpers::ConfigRetriever
+    #   config_retriever_for :start_from_beginning
+    # end
+    #
+    # Test.new.start_from_beginning #=> false
+    # test_instance = Test.new
+    # test_instance.start_from_beginning = true
+    # test_instance.start_from_beginning #=> true
+    module ConfigRetriever
+      # Builds proper methods for setting and retrieving (with fallback) given attribute value
+      # @param attribute [Symbol] attribute name based on which we will build
+      #   accessor with fallback
+      def config_retriever_for(attribute)
+        attr_writer attribute unless method_defined? :"#{attribute}="
+
+        # Don't redefine if we already have accessor for a given element
+        return if method_defined? attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+          return current_value unless current_value.nil?
+
+          value = if Karafka::App.config.respond_to?(attribute)
+                    Karafka::App.config.send(attribute)
+                  else
+                    Karafka::App.config.kafka.send(attribute)
+                  end
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/inflector.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Inflector provides inflection for the whole Karafka framework with additional inflection
+    # caching (due to the fact, that Dry::Inflector is slow)
+    module Inflector
+      # What inflection engine do we want to use
+      ENGINE = Dry::Inflector.new
+
+      @map = Concurrent::Hash.new
+
+      private_constant :ENGINE
+
+      class << self
+        # @param string [String] string that we want to convert to our underscore format
+        # @return [String] inflected string
+        # @example
+        #   Karafka::Helpers::Inflector.map('Module/ControllerName') #=> 'module_controller_name'
+        def map(string)
+          @map[string] ||= ENGINE.underscore(string).tr('/', '_')
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module containing classes and methods that provide some additional functionalities
+  module Helpers
+    # @note Taken from http://stackoverflow.com/questions/6407141
+    # Multidelegator is used to delegate calls to multiple targets
+    class MultiDelegator
+      # @param targets to which we want to delegate methods
+      def initialize(*targets)
+        @targets = targets
+      end
+
+      class << self
+        # @param methods names that should be delegated to
+        # @example Delegate write and close to STDOUT and file
+        #   Logger.new MultiDelegator.delegate(:write, :close).to(STDOUT, log_file)
+        def delegate(*methods)
+          methods.each do |m|
+            define_method(m) do |*args|
+              @targets.map { |t| t.send(m, *args) }
+            end
+          end
+
+          self
+        end
+
+        alias to new
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/logger.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default logger for Event Delegator
+    # @note It uses ::Logger features - providing basic logging
+    class Logger < ::Logger
+      # Map containing information about log level for given environment
+      ENV_MAP = {
+        'production' => ::Logger::ERROR,
+        'test' => ::Logger::ERROR,
+        'development' => ::Logger::INFO,
+        'debug' => ::Logger::DEBUG,
+        'default' => ::Logger::INFO
+      }.freeze
+
+      private_constant :ENV_MAP
+
+      # Creates a new instance of logger ensuring that it has a place to write to
+      # @param _args Any arguments that we don't care about but that are needed in order to
+      #   make this logger compatible with the default Ruby one
+      def initialize(*_args)
+        ensure_dir_exists
+        super(target)
+        self.level = ENV_MAP[Karafka.env] || ENV_MAP['default']
+      end
+
+      private
+
+      # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+      #   to which we will be writing logs
+      # We use this approach to log stuff to file and to the STDOUT at the same time
+      def target
+        Karafka::Helpers::MultiDelegator
+          .delegate(:write, :close)
+          .to(STDOUT, file)
+      end
+
+      # Makes sure the log directory exists as long as we can write to it
+      def ensure_dir_exists
+        FileUtils.mkdir_p(File.dirname(log_path))
+      rescue Errno::EACCES
+        nil
+      end
+
+      # @return [Pathname] Path to a file to which we should log
+      def log_path
+        @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
+      end
+
+      # @return [File] file to which we want to write our logs
+      # @note File is being opened in append mode ('a')
+      def file
+        @file ||= File.open(log_path, 'a')
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/monitor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related with Karafka instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how Karafka works
+    # It provides a standardized API for checking incoming messages/enqueueing etc
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and NewRelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Monitor < Dry::Monitor::Notifications
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      # @note Depending on Karafka extensions and additional engines, this might not be the
+      #   complete list of all the events. Please use the #available_events on fully loaded
+      #   Karafka system to determine all of the events you can use.
+      #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
+      #   whole karafka ecosystem
+      BASE_EVENTS = %w[
+        params.params.deserialize
+        params.params.deserialize.error
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
+        connection.listener.fetch_loop.error
+        connection.client.fetch_loop.error
+        connection.batch_delegator.call
+        connection.message_delegator.call
+        fetcher.call.error
+        backends.inline.process
+        process.notice_signal
+        consumers.responders.respond_with
+        async_producer.call.error
+        async_producer.call.retry
+        sync_producer.call.error
+        sync_producer.call.retry
+        app.initializing
+        app.initialized
+        app.running
+        app.stopping
+        app.stopping.error
+        app.stopped
+      ].freeze
+
+      private_constant :BASE_EVENTS
+
+      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super(:karafka)
+        BASE_EVENTS.each(&method(:register_event))
+      end
+
+      # Allows us to subscribe to events with a code that will be yielded upon events
+      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
+      #   or a listener if we decide to go with object listener
+      def subscribe(event_name_or_listener)
+        return super unless event_name_or_listener.is_a?(String)
+        return super if available_events.include?(event_name_or_listener)
+
+        raise Errors::UnregisteredMonitorEventError, event_name_or_listener
+      end
+
+      # @return [Array<String>] names of available events to which we can subscribe
+      def available_events
+        __bus__.events.keys
+      end
+    end
+  end
+end