karafka 0.5.0

data/lib/karafka/responders/usage_validator.rb

@@ -0,0 +1,59 @@
+module Karafka
+  module Responders
+    # Usage validator checks if all the requirements related to responders topics were met
+    class UsageValidator
+      # @param registered_topics [Hash] Hash with registered topics objects from
+      #   a given responder class under it's name key
+      # @param used_topics [Array<String>] Array with names of topics that we used in this
+      #   responding process
+      # @return [Karafka::Responders::UsageValidator] responding flow usage validator
+      def initialize(registered_topics, used_topics)
+        @registered_topics = registered_topics
+        @used_topics = used_topics
+      end
+
+      # Validates the whole flow
+      # @raise [Karafka::Errors::UnregisteredTopic] raised when we used a topic that we didn't
+      #   register using #topic method
+      # @raise [Karafka::Errors::TopicMultipleUsage] raised when we used a non multipleusage topic
+      #   multiple times
+      # @raise [Karafka::Errors::UnusedResponderRequiredTopic] raised when we didn't use a topic
+      #   that was defined as required to be used
+      def validate!
+        @used_topics.each do |used_topic|
+          validate_usage_of!(used_topic)
+        end
+
+        @registered_topics.each do |_name, registered_topic|
+          validate_requirements_of!(registered_topic)
+        end
+      end
+
+      private
+
+      # Checks if a given used topic were used in a proper way
+      # @raise [Karafka::Errors::UnregisteredTopic] raised when we used a topic that we didn't
+      #   register using #topic method
+      # @raise [Karafka::Errors::TopicMultipleUsage] raised when we used a non multipleusage topic
+      #   multiple times
+      # @param used_topic [String] topic to which we've sent a message
+      def validate_usage_of!(used_topic)
+        raise(Errors::UnregisteredTopic, used_topic) unless @registered_topics[used_topic]
+        return if @registered_topics[used_topic].multiple_usage?
+        return if @used_topics.count(used_topic) < 2
+        raise(Errors::TopicMultipleUsage, used_topic)
+      end
+
+      # Checks if we met all the requirements for all the registered topics
+      # @raise [Karafka::Errors::UnusedResponderRequiredTopic] raised when we didn't use a topic
+      #   that was defined as required to be used
+      # @param registered_topic [::Karafka::Responders::Topic] registered topic object
+      def validate_requirements_of!(registered_topic)
+        return unless registered_topic.required?
+        return if @used_topics.include?(registered_topic.name)
+
+        raise(Errors::UnusedResponderRequiredTopic, registered_topic.name)
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -0,0 +1,89 @@
+module Karafka
+  module Routing
+    # Routes builder used as a DSL layer for drawing and describing routes
+    # @example Build a simple (most common) route
+    #   draw do
+    #     topic :new_videos do
+    #       controller NewVideosController
+    #     end
+    #   end
+    class Builder < Array
+      include Singleton
+
+      # Options that are being set on the route level
+      ROUTE_OPTIONS = %i(
+        group
+        worker
+        controller
+        parser
+        interchanger
+        responder
+      ).freeze
+
+      # All those options should be set on the route level
+      ROUTE_OPTIONS.each do |option|
+        define_method option do |value|
+          @current_route.public_send :"#{option}=", value
+        end
+      end
+
+      # Creates a new route for a given topic and evalues provided block in builder context
+      # @param topic [String, Symbol] Kafka topic name
+      # @param block [Proc] block that will be evaluated in current context
+      # @note Creating new topic means creating a new route
+      # @example Define controller for a topic
+      #   topic :xyz do
+      #     controller XyzController
+      #   end
+      def topic(topic, &block)
+        @current_route = Route.new
+        @current_route.topic = topic
+
+        instance_eval(&block)
+
+        store!
+      end
+
+      # Used to draw routes for Karafka
+      # @note After it is done drawing it will store and validate all the routes to make sure that
+      #   they are correct and that there are no topic/group duplications (this is forbidden)
+      # @yield Evaluates provided block in a builder context so we can describe routes
+      # @example
+      #   draw do
+      #     topic :xyz do
+      #     end
+      #   end
+      def draw(&block)
+        instance_eval(&block)
+      end
+
+      private
+
+      # Stores current route locally after it was built and validated
+      def store!
+        @current_route.build
+        @current_route.validate!
+
+        self << @current_route
+
+        validate! :topic, Errors::DuplicatedTopicError
+        validate! :group, Errors::DuplicatedGroupError
+      end
+
+      # Checks that among all routes a given attribute value is unique
+      # @param attribute [Symbol] what routes attribute we want to check for uniqueness
+      # @param error [Class] error class that should be raised when something is wrong
+      def validate!(attribute, error)
+        map = each_with_object({}) do |route, amounts|
+          key = route.public_send(attribute)
+          amounts[key] = amounts[key].to_i + 1
+          amounts
+        end
+
+        wrong = map.find { |_, amount| amount > 1 }
+
+        raise error, wrong if wrong
+      end
+    end
+  end
+end

data/lib/karafka/routing/route.rb

@@ -0,0 +1,80 @@
+module Karafka
+  module Routing
+    # Class representing a single route (from topic to worker) with all additional features
+    # and elements. Single route contains descriptions of:
+    # - topic - Kafka topic name (required)
+    # - controller - Class of a controller that will handle messages from a given topic (required)
+    # - group - Kafka group that we want to use (optional)
+    # - worker - Which worker should handle the backend task (optional)
+    # - parser - What parsed do we want to use to unparse the data (optional)
+    # - interchanger - What interchanger to encode/decode data do we want to use (optional)
+    class Route
+      # Only ASCII alphanumeric characters and underscore and dash are allowed in topics and groups
+      NAME_FORMAT = /\A(\w|\-)+\z/
+
+      # Options that we can set per each route
+      attr_writer :group, :topic, :worker, :parser, :interchanger, :responder
+
+      # This we can get "directly" because it does not have any details, etc
+      attr_accessor :controller
+
+      # Initializes default values for all the options that support defaults if their values are
+      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
+      # everywhere except Karafka server command, those would not be initialized on time - for
+      # example for Sidekiq
+      def build
+        group
+        worker
+        parser
+        interchanger
+        responder
+        self
+      end
+
+      # @return [String] Kafka group name
+      # @note If group is not provided in a route, will build one based on the app name
+      #   and the route topic (that is required)
+      def group
+        (@group ||= "#{Karafka::App.config.name.underscore}_#{topic}").to_s
+      end
+
+      # @return [String] route topic - this is the core esence of Kafka
+      def topic
+        @topic.to_s
+      end
+
+      # @return [Class] Class (not an instance) of a worker that should be used to schedule the
+      #   background job
+      # @note If not provided - will be built based on the provided controller
+      def worker
+        @worker ||= Karafka::Workers::Builder.new(controller).build
+      end
+
+      # @return [Class, nil] Class (not an instance) of a responder that should respond from
+      #   controller back to Kafka (usefull for piping dataflows)
+      def responder
+        @responder ||= Karafka::Responders::Builder.new(controller).build
+      end
+
+      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
+      # @note If not provided - will use JSON as default
+      def parser
+        @parser ||= JSON
+      end
+
+      # @return [Class] Interchanger class (not an instance) that we want to use to interchange
+      #   params between Karafka server and Karafka background job
+      def interchanger
+        @interchanger ||= Karafka::Params::Interchanger
+      end
+
+      # Checks if topic and group have proper format (acceptable by Kafka)
+      # @raise [Karafka::Errors::InvalidTopicName] raised when topic name is invalid
+      # @raise [Karafka::Errors::InvalidGroupName] raised when group name is invalid
+      def validate!
+        raise Errors::InvalidTopicName, topic if NAME_FORMAT !~ topic
+        raise Errors::InvalidGroupName, group if NAME_FORMAT !~ group
+      end
+    end
+  end
+end

data/lib/karafka/routing/router.rb

@@ -0,0 +1,38 @@
+module Karafka
+  # Namespace for all elements related to requests routing
+  module Routing
+    # Karafka framework Router for routing incoming messages to proper controllers
+    # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
+    #  structure so all the routes are being stored in a single level array
+    class Router
+      # @param topic [String] topic based on which we find a proper route
+      # @return [Karafka::Router] router instance
+      def initialize(topic)
+        @topic = topic
+      end
+
+      # Builds a controller instance that should handle message from a given topic
+      # @return [Karafka::BaseController] base controller descendant instance object
+      def build
+        controller = route.controller.new
+        controller.topic = route.topic
+        controller.parser = route.parser
+        controller.worker = route.worker
+        controller.interchanger = route.interchanger
+        controller.responder = route.responder
+
+        controller
+      end
+
+      private
+
+      # @return [Karafka::Routing::Route] proper route details
+      # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
+      #   any route defined by user using routes.draw
+      def route
+        App.routes.find { |route| route.topic == @topic } ||
+          raise(Errors::NonMatchingRouteError, @topic)
+      end
+    end
+  end
+end

data/lib/karafka/server.rb

@@ -0,0 +1,53 @@
+module Karafka
+  # Karafka consuming server class
+  class Server
+    class << self
+      # We need to store reference to all the consumers in the main server thread,
+      # So we can have access to them later on and be able to stop them on exit
+      attr_reader :consumers
+
+      # Method which runs app
+      def run
+        @consumers = Concurrent::Array.new
+        bind_on_sigint
+        bind_on_sigquit
+        start_supervised
+      end
+
+      private
+
+      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
+      def process
+        Karafka::Process.instance
+      end
+
+      # What should happen when we decide to quit with sigint
+      def bind_on_sigint
+        process.on_sigint do
+          Karafka::App.stop!
+          consumers.map(&:stop) if Karafka::App.running?
+          exit
+        end
+      end
+
+      # What should happen when we decide to quit with sigquit
+      def bind_on_sigquit
+        process.on_sigquit do
+          Karafka::App.stop!
+          consumers.map(&:stop) if Karafka::App.running?
+          exit
+        end
+      end
+
+      # Starts Karafka with a supervision
+      # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
+      # finish loop (and it won't happen until we explicitily want to stop)
+      def start_supervised
+        process.supervise do
+          Karafka::App.run!
+          Karafka::Fetcher.new.fetch_loop
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -0,0 +1,57 @@
+module Karafka
+  # Module containing all Karafka setup related elements like configuration settings,
+  # config validations and configurators for external gems integration
+  module Setup
+    # Configurator for setting up all the framework details that are required to make it work
+    # @note If you want to do some configurations after all of this is done, please add to
+    #   karafka/config a proper file (needs to inherit from Karafka::Setup::Configurators::Base
+    #   and implement setup method) after that everything will happen automatically
+    # @note This config object allows to create a 1 level nestings (nodes) only. This should be
+    #   enough and will still keep the code simple
+    # @see Karafka::Setup::Configurators::Base for more details about configurators api
+    class Config
+      extend Dry::Configurable
+
+      # Available settings
+      # option name [String] current app name - used to provide default Kafka groups namespaces
+      setting :name
+      # option logger [Instance] logger that we want to use
+      setting :logger, ::Karafka::Logger.instance
+      # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
+      setting :monitor, ::Karafka::Monitor.instance
+      # option redis [Hash] redis options hash (url and optional parameters)
+      # Note that redis could be rewriten using nested options, but it is a sidekiq specific
+      # stuff and we don't want to touch it
+      setting :redis
+      # option kafka [Hash] - optional - kafka configuration options (hosts)
+      setting :kafka do
+        setting :hosts
+      end
+
+      # This is configured automatically, don't overwrite it!
+      # Each route requires separate thread, so number of threads should be equal to number
+      # of routes
+      setting :concurrency, -> { ::Karafka::App.routes.count }
+
+      class << self
+        # Configurating method
+        # @yield Runs a block of code providing a config singleton instance to it
+        # @yieldparam [Karafka::Setup::Config] Karafka config instance
+        def setup
+          configure do |config|
+            yield(config)
+          end
+        end
+
+        # Everything that should be initialized after the setup
+        # Components are in karafka/config directory and are all loaded one by one
+        # If you want to configure a next component, please add a proper file to config dir
+        def setup_components
+          Configurators::Base.descendants.each do |klass|
+            klass.new(config).setup
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/base.rb

@@ -0,0 +1,33 @@
+module Karafka
+  module Setup
+    # Configurators module is used to enclose all the external dependencies configurations
+    class Configurators
+      # Karafka has come components that it relies on (like Celluloid or Sidekiq)
+      # We need to configure all of them only when the framework was set up.
+      # Any class that descends from this one will be automatically invoked upon setup (after it)
+      # @example Configure an Example class
+      #   class ExampleConfigurator < Base
+      #     def setup
+      #       ExampleClass.logger = Karafka.logger
+      #       ExampleClass.redis = config.redis
+      #     end
+      #   end
+      class Base
+        extend ActiveSupport::DescendantsTracker
+
+        attr_reader :config
+
+        # @param config [Karafka::Config] config instance
+        # @return [Karafka::Config::Base] configurator for a given component
+        def initialize(config)
+          @config = config
+        end
+
+        # This method needs to be implemented in a subclass
+        def setup
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/celluloid.rb

@@ -0,0 +1,20 @@
+module Karafka
+  module Setup
+    class Configurators
+      # Class responsible for setting up Celluloid settings
+      class Celluloid < Base
+        # How many seconds should we wait for actors (listeners) before forcefully shutting them
+        SHUTDOWN_TIME = 30
+
+        # Sets up a Karafka logger as celluloid logger
+        def setup
+          ::Celluloid.logger = ::Karafka.logger
+          # This is just a precaution - it should automatically close the current
+          # connection and shutdown actor - but in case it didn't (hanged, etc)
+          # we will kill it after waiting for some time
+          ::Celluloid.shutdown_timeout = SHUTDOWN_TIME
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/configurators/sidekiq.rb

@@ -0,0 +1,34 @@
+module Karafka
+  module Setup
+    class Configurators
+      # Class to configure all the Sidekiq settings based on Karafka settings
+      class Sidekiq < Base
+        # Sets up sidekiq client and server
+        def setup
+          setup_sidekiq_client
+          setup_sidekiq_server
+        end
+
+        private
+
+        # Configure sidekiq client
+        def setup_sidekiq_client
+          ::Sidekiq.configure_client do |sidekiq_config|
+            sidekiq_config.redis = config.redis.to_h.merge(
+              size: config.concurrency
+            )
+          end
+        end
+
+        # Configure sidekiq setorrver
+        def setup_sidekiq_server
+          ::Sidekiq.configure_server do |sidekiq_config|
+            # We don't set size for the server - this will be set automatically based
+            # on the Sidekiq concurrency level (Sidekiq not Karafkas)
+            sidekiq_config.redis = config.redis.to_h
+          end
+        end
+      end
+    end
+  end
+end