karafka 0.5.0.3 → 0.6.0.rc1

data/lib/karafka/parsers/json.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Module for all supported by default parsers for incoming/outgoing data
   module Parsers
@@ -8,8 +10,8 @@ module Karafka
       # @example
       #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
       def self.parse(content)
-        ::JSON.parse(content)
-      rescue ::JSON::ParserError => e
+        ::Yajl::Parser.parse(content)
+      rescue ::Yajl::ParseError => e
         raise ::Karafka::Errors::ParserError, e
       end
 

data/lib/karafka/patches/dry_configurable.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Namespace for patches of external gems/libraries
   module Patches

data/lib/karafka/process.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Class used to catch signals from ruby Signal class in order to manage Karafka shutdown
   # @note There might be only one process - this class is a singleton
@@ -5,9 +7,9 @@ module Karafka
     include Singleton
 
     # Signal types that we handle
-    HANDLED_SIGNALS = %i(
+    HANDLED_SIGNALS = %i[
       SIGINT SIGQUIT SIGTERM
-    ).freeze
+    ].freeze
 
     HANDLED_SIGNALS.each do |signal|
       # Assigns a callback that will happen when certain signal will be send

data/lib/karafka/responders/builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   # Responders namespace encapsulates all the internal responder implementation parts
   module Responders

data/lib/karafka/responders/topic.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   module Responders
     # Topic describes a single topic on which we want to respond with responding requirements
@@ -17,7 +19,6 @@ module Karafka
       def initialize(name, options)
         @name = name.to_s
         @options = options
-        validate!
       end
 
       # @return [Boolean] is this a required topic (if not, it is optional)
@@ -30,12 +31,19 @@ module Karafka
         @options[:multiple_usage] || false
       end
 
-      private
+      # @return [Boolean] was usage of this topic registered or not
+      def registered?
+        @options[:registered] == true
+      end
 
-      # Checks topic name with the same regexp as routing
-      # @raise [Karafka::Errors::InvalidTopicName] raised when topic name is invalid
-      def validate!
-        raise Errors::InvalidTopicName, name if Routing::Route::NAME_FORMAT !~ name
+      # @return [Hash] hash with this topic attributes and options
+      def to_h
+        {
+          name: name,
+          multiple_usage: multiple_usage?,
+          required: required?,
+          registered: registered?
+        }
       end
     end
   end

data/lib/karafka/routing/builder.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Karafka
   module Routing
-    # Routes builder used as a DSL layer for drawing and describing routes
+    # Builder used as a DSL layer for building consumers and telling them which topics to consume
     # @example Build a simple (most common) route
-    #   draw do
+    #   consumers do
     #     topic :new_videos do
     #       controller NewVideosController
     #     end
@@ -10,42 +12,6 @@ module Karafka
     class Builder < Array
       include Singleton
 
-      # Options that are being set on the route level
-      ROUTE_OPTIONS = %i(
-        group
-        worker
-        controller
-        parser
-        interchanger
-        responder
-        inline_mode
-        batch_mode
-      ).freeze
-
-      # All those options should be set on the route level
-      ROUTE_OPTIONS.each do |option|
-        define_method option do |value|
-          @current_route.public_send :"#{option}=", value
-        end
-      end
-
-      # Creates a new route for a given topic and evalues provided block in builder context
-      # @param topic [String, Symbol] Kafka topic name
-      # @param block [Proc] block that will be evaluated in current context
-      # @note Creating new topic means creating a new route
-      # @example Define controller for a topic
-      #   topic :xyz do
-      #     controller XyzController
-      #   end
-      def topic(topic, &block)
-        @current_route = Route.new
-        @current_route.topic = topic
-
-        instance_eval(&block)
-
-        store!
-      end
-
       # Used to draw routes for Karafka
       # @note After it is done drawing it will store and validate all the routes to make sure that
       #   they are correct and that there are no topic/group duplications (this is forbidden)
@@ -57,34 +23,31 @@ module Karafka
       #   end
       def draw(&block)
         instance_eval(&block)
+
+        each do |consumer_group|
+          hashed_group = consumer_group.to_h
+          validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
+          next if validation_result.success?
+          raise Errors::InvalidConfiguration, [validation_result.errors, hashed_group]
+        end
       end
 
       private
 
-      # Stores current route locally after it was built and validated
-      def store!
-        @current_route.build
-        @current_route.validate!
-
-        self << @current_route
-
-        validate! :topic, Errors::DuplicatedTopicError
-        validate! :group, Errors::DuplicatedGroupError
+      # Builds and saves given consumer group
+      # @param group_id [String, Symbol] name for consumer group
+      # @yield Evaluates a given block in a consumer group context
+      def consumer_group(group_id, &block)
+        consumer_group = ConsumerGroup.new(group_id.to_s)
+        self << Proxy.new(consumer_group, &block).target
       end
 
-      # Checks that among all routes a given attribute value is unique
-      # @param attribute [Symbol] what routes attribute we want to check for uniqueness
-      # @param error [Class] error class that should be raised when something is wrong
-      def validate!(attribute, error)
-        map = each_with_object({}) do |route, amounts|
-          key = route.public_send(attribute)
-          amounts[key] = amounts[key].to_i + 1
-          amounts
+      # @param topic_name [String, Symbol] name of a topic from which we want to consumer
+      # @yield Evaluates a given block in a topic context
+      def topic(topic_name, &block)
+        consumer_group(topic_name) do
+          topic(topic_name, &block).tap(&:build)
         end
-
-        wrong = map.find { |_, amount| amount > 1 }
-
-        raise error, wrong if wrong
       end
     end
   end

data/lib/karafka/routing/consumer_group.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Object used to describe a single consumer group that is going to subscribe to
+    # given topics
+    # It is a part of Karafka's DSL
+    class ConsumerGroup
+      extend Helpers::ConfigRetriever
+
+      attr_reader :topics
+      attr_reader :id
+
+      # @param id [String, Symbol] raw id of this consumer group. Raw means, that it does not
+      #   yet have an application client_id namespace, this will be added here by default.
+      #   We add it to make a multi-system development easier for people that don't use
+      #   kafka and don't understand the concept of consumer groups.
+      def initialize(id)
+        @id = "#{Karafka::App.config.client_id.to_s.underscore}_#{id}"
+        @topics = []
+      end
+
+      # Builds a topic representation inside of a current consumer group route
+      # @param name [String, Symbol] name of topic to which we want to subscribe
+      # @yield Evaluates a given block in a topic context
+      # @return [Karafka::Routing::Topic] newly built topic instance
+      def topic=(name, &block)
+        topic = Topic.new(name, self)
+        @topics << Proxy.new(topic, &block).target.tap(&:build)
+        @topics.last
+      end
+
+      Karafka::AttributesMap.consumer_group.each do |attribute|
+        config_retriever_for(attribute)
+      end
+
+      # Hashed version of consumer group that can be used for validation purposes
+      # @return [Hash] hash with consumer group attributes including serialized to hash
+      # topics inside of it.
+      def to_h
+        result = {
+          topics: topics.map(&:to_h),
+          id: id
+        }
+
+        Karafka::AttributesMap.consumer_group.each do |attribute|
+          result[attribute] = public_send(attribute)
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/karafka/routing/mapper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Karafka
   module Routing
     # Default routes mapper that does not remap things

data/lib/karafka/routing/proxy.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Proxy is used as a translation layer in between the DSL and raw topic and consumer group
+    # objects.
+    class Proxy
+      attr_reader :target
+
+      # We should proxy only non ? and = methods as we want to have a regular dsl
+      IGNORED_POSTFIXES = %w[
+        ?
+        =
+        !
+      ].freeze
+
+      # @param target [Object] target object to which we proxy any DSL call
+      # @yield Evaluates block in the proxy context
+      def initialize(target, &block)
+        @target = target
+        instance_eval(&block)
+      end
+
+      # Translates the no "=" DSL of routing into elements assignments on target
+      def method_missing(method_name, *arguments, &block)
+        return super unless respond_to_missing?(method_name)
+        @target.public_send(:"#{method_name}=", *arguments, &block)
+      end
+
+      # Tells whether or not a given element exists on the target
+      def respond_to_missing?(method_name, include_private = false)
+        return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
+        @target.respond_to?(:"#{method_name}=", include_private) || super
+      end
+    end
+  end
+end

data/lib/karafka/routing/router.rb

@@ -1,24 +1,18 @@
+# frozen_string_literal: true
+
 module Karafka
   # Namespace for all elements related to requests routing
   module Routing
     # Karafka framework Router for routing incoming messages to proper controllers
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
-    class Router
-      # @param topic [String] topic based on which we find a proper route
-      # @return [Karafka::Router] router instance
-      def initialize(topic)
-        @topic = topic
-      end
-
+    module Router
       # Builds a controller instance that should handle message from a given topic
+      # @param topic_id [String] topic based on which we find a proper route
       # @return [Karafka::BaseController] base controller descendant instance object
-      def build
-        route.controller.new.tap do |ctrl|
-          Karafka::Routing::Route::ATTRIBUTES.each do |attr|
-            ctrl.public_send(:"#{attr}=", route.public_send(attr))
-          end
-        end
+      def build(topic_id)
+        topic = find(topic_id)
+        topic.controller.new.tap { |ctrl| ctrl.topic = topic }
       end
 
       private
@@ -26,10 +20,18 @@ module Karafka
       # @return [Karafka::Routing::Route] proper route details
       # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
       #   any route defined by user using routes.draw
-      def route
-        App.routes.find { |route| route.topic == @topic } ||
-          raise(Errors::NonMatchingRouteError, @topic)
+      def find(topic_id)
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            return topic if topic.id == topic_id
+          end
+        end
+
+        raise(Errors::NonMatchingRouteError, topic_id)
       end
+
+      module_function :build
+      module_function :find
     end
   end
 end

data/lib/karafka/routing/topic.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Topic stores all the details on how we should interact with Kafka given topic
+    # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
+    # It is a part of Karafka's DSL
+    class Topic
+      extend Helpers::ConfigRetriever
+
+      attr_reader :id, :consumer_group
+      attr_accessor :controller
+
+      # @param [String, Symbol] name of a topic on which we want to listen
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
+      def initialize(name, consumer_group)
+        @name = name.to_s
+        @consumer_group = consumer_group
+        @attributes = {}
+        # @note We use identifier related to the consumer group that owns a topic, because from
+        #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
+        #   have same topic name across mutliple Kafkas
+        @id = "#{consumer_group.id}_#{@name}"
+      end
+
+      # Initializes default values for all the options that support defaults if their values are
+      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
+      # everywhere except Karafka server command, those would not be initialized on time - for
+      # example for Sidekiq
+      def build
+        Karafka::AttributesMap.topic.each { |attr| send(attr) }
+        self
+      end
+
+      # @return [Class] Class (not an instance) of a worker that should be used to schedule the
+      #   background job
+      # @note If not provided - will be built based on the provided controller
+      def worker
+        @worker ||= inline_processing ? nil : Karafka::Workers::Builder.new(controller).build
+      end
+
+      # @return [Class, nil] Class (not an instance) of a responder that should respond from
+      #   controller back to Kafka (usefull for piping dataflows)
+      def responder
+        @responder ||= Karafka::Responders::Builder.new(controller).build
+      end
+
+      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
+      # @note If not provided - will use Json as default
+      def parser
+        @parser ||= Karafka::Parsers::Json
+      end
+
+      # @return [Class] Interchanger class (not an instance) that we want to use to interchange
+      #   params between Karafka server and Karafka background job
+      def interchanger
+        @interchanger ||= Karafka::Params::Interchanger
+      end
+
+      Karafka::AttributesMap.topic.each do |attribute|
+        config_retriever_for(attribute)
+      end
+
+      # @return [Hash] hash with all the topic attributes
+      # @note This is being used when we validate the consumer_group and its topics
+      def to_h
+        map = Karafka::AttributesMap.topic.map do |attribute|
+          [attribute, public_send(attribute)]
+        end
+
+        Hash[map].merge!(
+          id: id,
+          controller: controller
+        )
+      end
+    end
+  end
+end

data/lib/karafka/schemas/config.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the validation schemas that we use to check input
+  module Schemas
+    # Regexp for validating format of groups and topics
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
+
+    # Schema with validation rules for Karafka configuration details
+    # @note There are many more configuration options inside of the
+    #   Karafka::Setup::Config model, but we don't validate them here as they are
+    #   validated per each route (topic + consumer_group) because they can be overwritten,
+    #   so we validate all of that once all the routes are defined and ready
+    Config = Dry::Validation.Schema do
+      required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+
+      required(:redis).maybe do
+        schema do
+          required(:url).filled(:str?)
+        end
+      end
+
+      optional(:inline_processing).filled(:bool?)
+
+      # If inline_processing is true, redis should be filled
+      rule(redis_presence: %i[redis inline_processing]) do |redis, inline_processing|
+        inline_processing.false?.then(redis.filled?)
+      end
+
+      optional(:connection_pool).schema do
+        required(:size).filled
+        optional(:timeout).filled(:int?)
+      end
+    end
+  end
+end