karafka 0.5.0

data/lib/karafka/cli/worker.rb

@@ -0,0 +1,26 @@
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Worker Karafka Cli action
+    class Worker < Base
+      desc 'Start the Karafka Sidekiq worker (short-cut alias: "w")'
+      option aliases: 'w'
+
+      # Start the Karafka Sidekiq worker
+      # @param params [Array<String>] additional params that will be passed to sidekiq, that way we
+      #   can override any default Karafka settings
+      def call(*params)
+        puts 'Starting Karafka worker'
+        config = "-C #{Karafka::App.root.join('config/sidekiq.yml')}"
+        req = "-r #{Karafka.boot_file}"
+        env = "-e #{Karafka.env}"
+
+        cli.info
+
+        cmd = "bundle exec sidekiq #{env} #{req} #{config} #{params.join(' ')}"
+        puts(cmd)
+        exec(cmd)
+      end
+    end
+  end
+end

data/lib/karafka/connection/consumer.rb

@@ -0,0 +1,29 @@
+module Karafka
+  module Connection
+    # Class that consumes messages for which we listen
+    class Consumer
+      # Consumes a message (does something with it)
+      # It will execute a scheduling task from a proper controller based on a message topic
+      # @note This should be looped to obtain a constant listening
+      # @note We catch all the errors here, to make sure that none failures
+      #   for a given consumption will affect other consumed messages
+      #   If we would't catch it, it would propagate up until killing the Celluloid actor
+      # @param message [Kafka::FetchedMessage] message that was fetched by kafka
+      def consume(message)
+        controller = Karafka::Routing::Router.new(message.topic).build
+        # We wrap it around with our internal message format, so we don't pass around
+        # a raw Kafka message
+        controller.params = Message.new(message.topic, message.value)
+
+        Karafka.monitor.notice(self.class, controller.to_h)
+
+        controller.schedule
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        # rubocop:enable RescueException
+        Karafka.monitor.notice_error(self.class, e)
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -0,0 +1,54 @@
+module Karafka
+  module Connection
+    # A single listener that listens to incoming messages from a single route
+    # @note It does not loop on itself - it needs to be executed in a loop
+    # @note Listener itself does nothing with the message - it will return to the block
+    #   a raw Kafka::FetchedMessage
+    class Listener
+      include Celluloid
+
+      execute_block_on_receiver :fetch_loop
+
+      attr_reader :route
+
+      # @return [Karafka::Connection::Listener] listener instance
+      def initialize(route)
+        @route = route
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Karafka::BaseController] base controller descendant
+      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
+      # @note This will yield with a raw message - no preprocessing or reformatting
+      # @note We catch all the errors here, so they don't affect other listeners (or this one)
+      #   so we will be able to listen and consume other incoming messages.
+      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
+      #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
+      #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
+      #   propagate this far
+      def fetch_loop(block)
+        topic_consumer.fetch_loop do |raw_message|
+          block.call(raw_message)
+        end
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        # rubocop:enable RescueException
+        Karafka.monitor.notice_error(self.class, e)
+        @topic_consumer&.stop
+        retry if @topic_consumer
+      end
+
+      private
+
+      # @return [Karafka::Connection::TopicConsumer] wrapped kafka consumer for a given topic
+      #   consumption
+      # @note It adds consumer into Karafka::Server consumers pool for graceful shutdown on exit
+      def topic_consumer
+        @topic_consumer ||= TopicConsumer.new(@route).tap do |consumer|
+          Karafka::Server.consumers << consumer if Karafka::Server.consumers
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/message.rb

@@ -0,0 +1,17 @@
+module Karafka
+  # Namespace that encapsulates everything related to connections
+  module Connection
+    # Single incoming Kafka message instance wrapper
+    class Message
+      attr_reader :topic, :content
+
+      # @param topic [String] topic from which this message comes
+      # @param content [String] raw message content (not deserialized or anything) from Kafka
+      # @return [Karafka::Connection::Message] incoming message instance
+      def initialize(topic, content)
+        @topic = topic
+        @content = content
+      end
+    end
+  end
+end

data/lib/karafka/connection/topic_consumer.rb

@@ -0,0 +1,48 @@
+module Karafka
+  module Connection
+    # Class used as a wrapper around Ruby-Kafka to simplify additional
+    # features that we provide/might provide in future
+    class TopicConsumer
+      # Creates a queue consumer that will pull the data from Kafka
+      # @param [Karafka::Routing::Route] route details that will be used to build up a
+      #   queue consumer instance
+      # @return [Karafka::Connection::QueueConsumer] queue consumer instance
+      def initialize(route)
+        @route = route
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
+      # @note This will yield with a raw message - no preprocessing or reformatting
+      def fetch_loop
+        kafka_consumer.each_message do |message|
+          yield(message)
+        end
+      end
+
+      # Gracefuly stops topic consumption
+      def stop
+        kafka_consumer.stop
+        @kafka_consumer = nil
+      end
+
+      private
+
+      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
+      #   that is set up to consume a given routes topic
+      def kafka_consumer
+        return @kafka_consumer if @kafka_consumer
+
+        kafka = Kafka.new(
+          seed_brokers: ::Karafka::App.config.kafka.hosts,
+          logger: ::Karafka.logger,
+          client_id: ::Karafka::App.config.name
+        )
+
+        @kafka_consumer = kafka.consumer(group_id: @route.group)
+        @kafka_consumer.subscribe(@route.topic)
+        @kafka_consumer
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -0,0 +1,50 @@
+module Karafka
+  # Namespace used to encapsulate all the internal errors of Karafka
+  module Errors
+    # Base class for all the Karafka internal errors
+    class BaseError < StandardError; end
+
+    # Should be raised when we attemp to parse incoming params but parsing fails
+    #   If this error (or its descendant) is detected, we will pass the raw message
+    #   into params and proceed further
+    class ParserError < BaseError; end
+
+    # Raised when router receives topic name which does not correspond with any routes
+    #   This should never happen because we listed only to topics defined in routes
+    #   but theory is not always right. If you encounter this error - please contact
+    #   Karafka maintainers
+    class NonMatchingRouteError < BaseError; end
+
+    # Raised when we have few controllers(inherited from Karafka::BaseController)
+    #   with the same group name
+    class DuplicatedGroupError < BaseError; end
+
+    # Raised when we have few controllers(inherited from Karafka::BaseController)
+    #   with the same topic name
+    class DuplicatedTopicError < BaseError; end
+
+    # Raised when we want to use topic name that has unsupported characters
+    class InvalidTopicName < BaseError; end
+
+    # Raised when we want to use group name that has unsupported characters
+    class InvalidGroupName < BaseError; end
+
+    # Raised when application does not have ApplicationWorker or other class that directly
+    # inherits from Karafka::BaseWorker
+    class BaseWorkerDescentantMissing < BaseError; end
+
+    # Raised when we want to use #respond_with in controllers but we didn't define
+    # (and we couldn't find) any appropriate responder for a given controller
+    class ResponderMissing < BaseError; end
+
+    # Raised when we want to use #respond_to in responders with a topic that we didn't register
+    class UnregisteredTopic < BaseError; end
+
+    # Raised when we send more than one message to a single topic but we didn't allow that when
+    # we were registering topic in a responder
+    class TopicMultipleUsage < BaseError; end
+
+    # Raised when we didn't use a topic that was defined as non-optional (required)
+    class UnusedResponderRequiredTopic < BaseError; end
+  end
+end

data/lib/karafka/fetcher.rb

@@ -0,0 +1,40 @@
+module Karafka
+  # Class used to run the Karafka consumer and handle shutting down, restarting etc
+  # @note Creating multiple fetchers will result in having multiple connections to the same
+  #   topics, which means that if there are no partitions, it won't use them.
+  class Fetcher
+    # Starts listening on all the listeners asynchronously
+    # Fetch loop should never end, which means that we won't create more actor clusters
+    # so we don't have to terminate them
+    def fetch_loop
+      futures = listeners.map do |listener|
+        listener.future.public_send(:fetch_loop, consumer)
+      end
+
+      futures.map(&:value)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something really bad happened
+    rescue => e
+      Karafka.monitor.notice_error(self.class, e)
+      Karafka::App.stop!
+      raise e
+    end
+
+    private
+
+    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+    def listeners
+      @listeners ||= App.routes.map do |route|
+        Karafka::Connection::Listener.new(route)
+      end
+    end
+
+    # @return [Proc] proc that should be processed when a message arrives
+    # @yieldparam message [Kafka::FetchedMessage] message from kafka (raw one)
+    def consumer
+      lambda do |message|
+        Karafka::Connection::Consumer.new.consume(message)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/class_matcher.rb

@@ -0,0 +1,77 @@
+module Karafka
+  module Helpers
+    # Class used to autodetect corresponding classes that are internally inside Karafka framework
+    # It is used among others to match:
+    #   controller => worker
+    #   controller => responder
+    class ClassMatcher
+      # Regexp used to remove any non classy like characters that might be in the controller
+      # class name (if defined dynamically, etc)
+      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
+
+      # @param klass [Class] class to which we want to find a corresponding class
+      # @param from [String] what type of object is it (based on postfix name part)
+      # @param to [String] what are we looking for (based on a postfix name part)
+      # @example Controller that has a corresponding worker
+      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperController, 'Controller', 'Worker')
+      #   matcher.match #=> SuperWorker
+      # @example Controller without a corresponding worker
+      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Controller, 'Controller', 'Worker')
+      #   matcher.match #=> nil
+      def initialize(klass, from:, to:)
+        @klass = klass
+        @from = from
+        @to = to
+      end
+
+      # @return [Class] matched class
+      # @return [nil] nil if we couldn't find matching class
+      def match
+        return nil if name.empty?
+        return nil unless scope.const_defined?(name)
+        matching = scope.const_get(name)
+        same_scope?(matching) ? matching : nil
+      end
+
+      # @return [String] name of a new class that we're looking for
+      # @note This method returns name of a class without a namespace
+      # @example From SuperController matching worker
+      #   matcher.name #=> 'SuperWorker'
+      # @example From Namespaced::Super2Controller matching worker
+      #   matcher.name #=> Super2Worker
+      def name
+        inflected = @klass.to_s.split('::').last.to_s
+        inflected.gsub!(@from, @to)
+        inflected.gsub!(CONSTANT_REGEXP, '')
+        inflected
+      end
+
+      # @return [Class, Module] class or module in which we're looking for a matching
+      def scope
+        scope_of(@klass)
+      end
+
+      private
+
+      # @param klass [Class] class for which we want to extract it's enclosing class/module
+      # @return [Class, Module] enclosing class/module
+      # @return [::Object] object if it was a root class
+      #
+      # @example Non-namespaced class
+      #   scope_of(SuperClass) #=> Object
+      # @example Namespaced class
+      #   scope_of(Abc::SuperClass) #=> Abc
+      def scope_of(klass)
+        enclosing = klass.to_s.split('::')[0...-1]
+        return ::Object if enclosing.empty?
+        ::Object.const_get(enclosing.join('::'))
+      end
+
+      # @param matching [Class] class of which scope we want to check
+      # @return [Boolean] true if the scope of class is the same as scope of matching
+      def same_scope?(matching)
+        scope == scope_of(matching)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -0,0 +1,31 @@
+module Karafka
+  # Module containing classes and methods that provide some additional functionalities
+  module Helpers
+    # @note Taken from http://stackoverflow.com/questions/6407141
+    # Multidelegator is used to delegate calls to multiple targets
+    class MultiDelegator
+      # @param targets to which we want to delegate methods
+      #
+      def initialize(*targets)
+        @targets = targets
+      end
+
+      class << self
+        # @param methods names that should be delegated to
+        # @example Delegate write and close to STDOUT and file
+        #   Logger.new MultiDelegator.delegate(:write, :close).to(STDOUT, log_file)
+        def delegate(*methods)
+          methods.each do |m|
+            define_method(m) do |*args|
+              @targets.map { |t| t.send(m, *args) }
+            end
+          end
+
+          self
+        end
+
+        alias to new
+      end
+    end
+  end
+end

data/lib/karafka/loader.rb

@@ -0,0 +1,77 @@
+module Karafka
+  # Loader for requiring all the files in a proper order
+  # Some files needs to be required before other, so it will
+  # try to figure it out. It will load 'base*' files first and then
+  # any other.
+  class Loader
+    # Order in which we want to load app files
+    DIRS = %w(
+      config/initializers
+      lib
+      app/helpers
+      app/inputs
+      app/decorators
+      app/models/concerns
+      app/models
+      app/responders
+      app/services
+      app/presenters
+      app/workers
+      app/controllers
+      app/aspects
+      app
+    ).freeze
+
+    # Will load files in a proper order (based on DIRS)
+    # @param [String] root path from which we want to start
+    def load(root)
+      DIRS.each do |dir|
+        path = File.join(root, dir)
+        load!(path)
+      end
+    end
+
+    # Requires all the ruby files from one path in a proper order
+    # @param path [String] path (dir) from which we want to load ruby files in a proper order
+    # @note First we load all the base files that might be used in inheritance
+    def load!(path)
+      base_load!(path)
+      files_load!(path)
+    end
+
+    # Requires all the ruby files from one relative path inside application directory
+    # @param relative_path [String] relative path (dir) to a file inside application directory
+    #   from which we want to load ruby files in a proper order
+    def relative_load!(relative_path)
+      path = File.join(::Karafka.root, relative_path)
+      load!(path)
+    end
+
+    private
+
+    # Loads all the base files
+    # @param path [String] path (dir) from which we want to load ruby base files in a proper order
+    def base_load!(path)
+      bases = File.join(path, '**/base*.rb')
+      Dir[bases].sort(&method(:base_sorter)).each(&method(:require))
+    end
+
+    # Loads all other files (not base)
+    # @param path [String] path (dir) from which we want to load ruby files in a proper order
+    # @note Technically it will load the base files again but they are already loaded so nothing
+    #   will happen
+    def files_load!(path)
+      files = File.join(path, '**/*.rb')
+      Dir[files].sort.each(&method(:require))
+    end
+
+    # @return [Integer] order for sorting
+    # @note We need sort all base files based on their position in a file tree
+    #   so all the files that are "higher" should be loaded first
+    # @param str1 [String] first string for comparison
+    # @param str2 [String] second string for comparison
+    def base_sorter(str1, str2)
+      str1.count('/') <=> str2.count('/')
+    end
+  end
+end