karafka 0.5.0.3 → 0.6.0.rc1

data/lib/karafka/cli/routes.rb

@@ -1,36 +0,0 @@
-module Karafka
-  # Karafka framework Cli
-  class Cli
-    # Routes Karafka Cli action
-    class Routes < Base
-      desc 'Print out all defined routes in alphabetical order'
-      option aliases: 'r'
-
-      # Print out all defined routes in alphabetical order
-      def call
-        routes.each do |route|
-          puts "#{route.topic}:"
-          Karafka::Routing::Route::ATTRIBUTES.each do |attr|
-            print(attr.to_s.capitalize, route.public_send(attr))
-          end
-        end
-      end
-
-      private
-
-      # @return [Array<Karafka::Routing::Route>] all routes sorted in alphabetical order
-      def routes
-        Karafka::App.routes.sort do |route1, route2|
-          route1.topic <=> route2.topic
-        end
-      end
-
-      # Prints a given value with label in a nice way
-      # @param label [String] label describing value
-      # @param value [String] value that should be printed
-      def print(label, value)
-        printf "%-18s %s\n", "  - #{label}:", value
-      end
-    end
-  end
-end

data/lib/karafka/connection/consumer.rb

@@ -1,33 +0,0 @@
-module Karafka
-  module Connection
-    # Class that consumes messages for which we listen
-    class Consumer
-      # Consumes a message (does something with it)
-      # It will execute a scheduling task from a proper controller based on a message topic
-      # @note This should be looped to obtain a constant listening
-      # @note We catch all the errors here, to make sure that none failures
-      #   for a given consumption will affect other consumed messages
-      #   If we would't catch it, it would propagate up until killing the Celluloid actor
-      # @param message [Kafka::FetchedMessage] message that was fetched by kafka
-      def consume(message)
-        # We map from incoming topic name, as it might be namespaced, etc.
-        # @see topic_mapper internal docs
-        mapped_topic = Karafka::App.config.topic_mapper.incoming(message.topic)
-
-        controller = Karafka::Routing::Router.new(mapped_topic).build
-        # We wrap it around with our internal message format, so we don't pass around
-        # a raw Kafka message
-        controller.params = Message.new(mapped_topic, message.value)
-
-        Karafka.monitor.notice(self.class, controller.to_h)
-
-        controller.schedule
-        # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
-      rescue Exception => e
-        # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
-      end
-    end
-  end
-end

data/lib/karafka/connection/message.rb

@@ -1,17 +0,0 @@
-module Karafka
-  # Namespace that encapsulates everything related to connections
-  module Connection
-    # Single incoming Kafka message instance wrapper
-    class Message
-      attr_reader :topic, :content
-
-      # @param topic [String] topic from which this message comes
-      # @param content [String] raw message content (not deserialized or anything) from Kafka
-      # @return [Karafka::Connection::Message] incoming message instance
-      def initialize(topic, content)
-        @topic = topic
-        @content = content
-      end
-    end
-  end
-end

data/lib/karafka/connection/topic_consumer.rb

@@ -1,94 +0,0 @@
-module Karafka
-  module Connection
-    # Class used as a wrapper around Ruby-Kafka to simplify additional
-    # features that we provide/might provide in future
-    class TopicConsumer
-      # How long should we wait before trying to reconnect to Kafka cluster
-      # that went down (in seconds)
-      RECONNECT_TIMEOUT = 5
-
-      # Creates a queue consumer that will pull the data from Kafka
-      # @param [Karafka::Routing::Route] route details that will be used to build up a
-      #   queue consumer instance
-      # @return [Karafka::Connection::QueueConsumer] queue consumer instance
-      def initialize(route)
-        @route = route
-      end
-
-      # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
-      # @note This will yield with a raw message - no preprocessing or reformatting
-      def fetch_loop
-        send(
-          @route.batch_mode ? :consume_each_batch : :consume_each_message
-        ) do |message|
-          yield(message)
-        end
-      end
-
-      # Gracefuly stops topic consumption
-      def stop
-        @kafka_consumer&.stop
-        @kafka_consumer = nil
-      end
-
-      private
-
-      # Consumes messages from Kafka in batches
-      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
-      def consume_each_batch
-        kafka_consumer.each_batch do |batch|
-          batch.messages.each do |message|
-            yield(message)
-          end
-        end
-      end
-
-      # Consumes messages from Kafka one by one
-      # @yieldparam [Kafka::FetchedMessage] kafka fetched message
-      def consume_each_message
-        kafka_consumer.each_message do |message|
-          yield(message)
-        end
-      end
-
-      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
-      #   that is set up to consume a given routes topic
-      def kafka_consumer
-        @kafka_consumer ||= kafka.consumer(
-          group_id: @route.group,
-          session_timeout: ::Karafka::App.config.kafka.session_timeout,
-          offset_commit_interval: ::Karafka::App.config.kafka.offset_commit_interval,
-          offset_commit_threshold: ::Karafka::App.config.kafka.offset_commit_threshold,
-          heartbeat_interval: ::Karafka::App.config.kafka.heartbeat_interval
-        ).tap do |consumer|
-          consumer.subscribe(
-            @route.topic,
-            start_from_beginning: @route.start_from_beginning
-          )
-        end
-      rescue Kafka::ConnectionError
-        # If we would not wait it would totally spam log file with failed
-        # attempts if Kafka is down
-        sleep(RECONNECT_TIMEOUT)
-        # We don't log and just reraise - this will be logged
-        # down the road
-        raise
-      end
-
-      # @return [Kafka] returns a Kafka
-      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
-      #   object instance
-      def kafka
-        Kafka.new(
-          seed_brokers: ::Karafka::App.config.kafka.hosts,
-          logger: ::Karafka.logger,
-          client_id: ::Karafka::App.config.name,
-          ssl_ca_cert: ::Karafka::App.config.kafka.ssl.ca_cert,
-          ssl_client_cert: ::Karafka::App.config.kafka.ssl.client_cert,
-          ssl_client_cert_key: ::Karafka::App.config.kafka.ssl.client_cert_key
-        )
-      end
-    end
-  end
-end

data/lib/karafka/responders/usage_validator.rb

@@ -1,60 +0,0 @@
-module Karafka
-  module Responders
-    # Usage validator checks if all the requirements related to responders topics were met
-    class UsageValidator
-      # @param registered_topics [Hash] Hash with registered topics objects from
-      #   a given responder class under it's name key
-      # @param used_topics [Array<String>] Array with names of topics that we used in this
-      #   responding process
-      # @return [Karafka::Responders::UsageValidator] responding flow usage validator
-      def initialize(registered_topics, used_topics)
-        @registered_topics = registered_topics
-        @used_topics = used_topics
-      end
-
-      # Validates the whole flow
-      # @raise [Karafka::Errors::UnregisteredTopic] raised when we used a topic that we didn't
-      #   register using #topic method
-      # @raise [Karafka::Errors::TopicMultipleUsage] raised when we used a non multipleusage topic
-      #   multiple times
-      # @raise [Karafka::Errors::UnusedResponderRequiredTopic] raised when we didn't use a topic
-      #   that was defined as required to be used
-      def validate!
-        @used_topics.each do |used_topic|
-          validate_usage_of!(used_topic)
-        end
-
-        @registered_topics.each do |_name, registered_topic|
-          validate_requirements_of!(registered_topic)
-        end
-      end
-
-      private
-
-      # Checks if a given used topic were used in a proper way
-      # @raise [Karafka::Errors::UnregisteredTopic] raised when we used a topic that we didn't
-      #   register using #topic method
-      # @raise [Karafka::Errors::TopicMultipleUsage] raised when we used a non multipleusage topic
-      #   multiple times
-      # @param used_topic [String] topic to which we've sent a message
-      def validate_usage_of!(used_topic)
-        raise(Errors::UnregisteredTopic, used_topic) unless @registered_topics[used_topic]
-        return if @registered_topics[used_topic].multiple_usage?
-        return unless @registered_topics[used_topic].required?
-        return if @used_topics.count(used_topic) < 2
-        raise(Errors::TopicMultipleUsage, used_topic)
-      end
-
-      # Checks if we met all the requirements for all the registered topics
-      # @raise [Karafka::Errors::UnusedResponderRequiredTopic] raised when we didn't use a topic
-      #   that was defined as required to be used
-      # @param registered_topic [::Karafka::Responders::Topic] registered topic object
-      def validate_requirements_of!(registered_topic)
-        return unless registered_topic.required?
-        return if @used_topics.include?(registered_topic.name)
-
-        raise(Errors::UnusedResponderRequiredTopic, registered_topic.name)
-      end
-    end
-  end
-end

data/lib/karafka/routing/route.rb

@@ -1,113 +0,0 @@
-module Karafka
-  module Routing
-    # Class representing a single route (from topic to worker) with all additional features
-    # and elements. Single route contains descriptions of:
-    # - topic - Kafka topic name (required)
-    # - controller - Class of a controller that will handle messages from a given topic (required)
-    # - group - Kafka group that we want to use (optional)
-    # - worker - Which worker should handle the backend task (optional)
-    # - parser - What parsed do we want to use to unparse the data (optional)
-    # - interchanger - What interchanger to encode/decode data do we want to use (optional)
-    class Route
-      # Only ASCII alphanumeric characters, underscore, dash and dots
-      # are allowed in topics and groups
-      NAME_FORMAT = /\A(\w|\-|\.)+\z/
-
-      # Options that we can set per each route
-      ATTRIBUTES = %i(
-        group
-        topic
-        worker
-        parser
-        interchanger
-        responder
-        inline_mode
-        batch_mode
-        start_from_beginning
-      ).freeze
-
-      ATTRIBUTES.each { |attr| attr_writer(attr) }
-
-      # This we can get "directly" because it does not have any details, etc
-      attr_accessor :controller
-
-      # Initializes default values for all the options that support defaults if their values are
-      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
-      # everywhere except Karafka server command, those would not be initialized on time - for
-      # example for Sidekiq
-      def build
-        ATTRIBUTES.each { |attr| send(attr) }
-        self
-      end
-
-      # @return [String] Kafka group name
-      # @note If group is not provided in a route, will build one based on the app name
-      #   and the route topic (that is required)
-      def group
-        (@group ||= "#{Karafka::App.config.name.underscore}_#{topic}").to_s
-      end
-
-      # @return [String] route topic - this is the core esence of Kafka
-      def topic
-        @topic.to_s
-      end
-
-      # @return [Class] Class (not an instance) of a worker that should be used to schedule the
-      #   background job
-      # @note If not provided - will be built based on the provided controller
-      def worker
-        @worker ||= inline_mode ? nil : Karafka::Workers::Builder.new(controller).build
-      end
-
-      # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   controller back to Kafka (usefull for piping dataflows)
-      def responder
-        @responder ||= Karafka::Responders::Builder.new(controller).build
-      end
-
-      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
-      # @note If not provided - will use Json as default
-      def parser
-        @parser ||= Karafka::Parsers::Json
-      end
-
-      # @return [Class] Interchanger class (not an instance) that we want to use to interchange
-      #   params between Karafka server and Karafka background job
-      def interchanger
-        @interchanger ||= Karafka::Params::Interchanger
-      end
-
-      # @return [Boolean] Should we perform execution in the background (default) or
-      #   inline. This can be set globally and overwritten by a per route setting
-      # @note This method can be set to false, so direct assigment ||= would not work
-      def inline_mode
-        return @inline_mode unless @inline_mode.nil?
-        @inline_mode = Karafka::App.config.inline_mode
-      end
-
-      # @return [Boolean] Should the consumer handle incoming events one at a time, or in batch
-      def batch_mode
-        return @batch_mode unless @batch_mode.nil?
-        @batch_mode = Karafka::App.config.batch_mode
-      end
-
-      # For each topic subscription it's possible to decide whether to consume messages starting
-      # at the beginning of the topic or to just consume new messages that are produced to
-      # the topic.
-      # @return [Boolean] Should we consume from the beggining or from new incoming messages on
-      #   the first run
-      def start_from_beginning
-        return @start_from_beginning unless @start_from_beginning.nil?
-        @start_from_beginning = Karafka::App.config.start_from_beginning
-      end
-
-      # Checks if topic and group have proper format (acceptable by Kafka)
-      # @raise [Karafka::Errors::InvalidTopicName] raised when topic name is invalid
-      # @raise [Karafka::Errors::InvalidGroupName] raised when group name is invalid
-      def validate!
-        raise Errors::InvalidTopicName, topic if NAME_FORMAT !~ topic
-        raise Errors::InvalidGroupName, group if NAME_FORMAT !~ group
-      end
-    end
-  end
-end

data/lib/karafka/setup/config_schema.rb

@@ -1,44 +0,0 @@
-module Karafka
-  module Setup
-    # Schema with validation rules for all configuration
-    ConfigSchema = Dry::Validation.Schema do
-      required(:name).filled(:str?)
-      required(:topic_mapper).filled
-      optional(:inline_mode).filled(:bool?)
-
-      required(:redis).maybe do
-        schema do
-          required(:url).filled(:str?)
-        end
-      end
-
-      # If inline_mode is true, redis should be filled
-      rule(redis_presence: [:redis, :inline_mode]) do |redis, inline_mode|
-        inline_mode.false?.then(redis.filled?)
-      end
-
-      optional(:batch_mode).filled(:bool?)
-      optional(:start_from_beginning).filled(:bool?)
-
-      optional(:connection_pool).schema do
-        required(:size).filled
-        optional(:timeout).filled(:int?)
-      end
-
-      required(:kafka).schema do
-        required(:hosts).filled(:array?)
-
-        required(:session_timeout).filled(:int?)
-        required(:offset_commit_interval).filled(:int?)
-        required(:offset_commit_threshold).filled(:int?)
-        required(:heartbeat_interval).filled(:int?)
-
-        optional(:ssl).schema do
-          required(:ca_cert).maybe(:str?)
-          required(:client_cert).maybe(:str?)
-          required(:client_cert_key).maybe(:str?)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/setup/configurators/worker_glass.rb

@@ -1,13 +0,0 @@
-module Karafka
-  module Setup
-    class Configurators
-      # Class responsible for setting up WorkerGlass settings
-      class WorkerGlass < Base
-        # Sets up a Karafka logger as celluloid logger
-        def setup
-          ::WorkerGlass.logger = ::Karafka.logger
-        end
-      end
-    end
-  end
-end

data/lib/karafka/templates/config.ru.example

@@ -1,13 +0,0 @@
-%w(
-  puma
-  sidekiq/web
-).each { |lib| require lib }
-
-require Karafka.boot_file
-
-use Rack::Auth::Basic, 'Protected Area' do |username, password|
-  username == 'sidekiq' &&
-    password == 'Pa$$WorD!'
-end
-
-run Sidekiq::Web