karafka 1.2.0

data/lib/karafka/params/params_batch.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Params batch represents a set of messages received from Kafka.
+    # @note Params internally are lazy loaded before first use. That way we can skip parsing
+    #   process if we have after_fetch that rejects some incoming messages without using params
+    #   It can be also used when handling really heavy data (in terms of parsing).
+    class ParamsBatch
+      include Enumerable
+
+      # Builds up a params batch based on raw kafka messages
+      # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
+      # @param topic_parser [Class] topic parser for unparsing messages values
+      def initialize(messages_batch, topic_parser)
+        @params_batch = messages_batch.map! do |message|
+          Karafka::Params::Params.build(message, topic_parser)
+        end
+      end
+
+      # @yieldparam [Karafka::Params::Params] each parsed and loaded params instance
+      # @note Invocation of this method will cause loading and parsing each param after another.
+      #   If you want to get access without parsing, please access params_batch directly
+      def each
+        @params_batch.each { |param| yield(param.retrieve!) }
+      end
+
+      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
+      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
+      #   be parsed
+      def parsed
+        each(&:itself)
+      end
+
+      # @return [Karafka::Params::Params] last element after the unparsing process
+      def last
+        @params_batch.last.retrieve!
+      end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
+      def to_a
+        @params_batch
+      end
+    end
+  end
+end

data/lib/karafka/parsers/json.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default parsers for incoming/outgoing data
+  module Parsers
+    # Default Karafka Json parser for serializing and deserializing data
+    class Json
+      # @param content [String] content based on which we want to get our hash
+      # @return [Hash] hash with parsed JSON data
+      # @example
+      #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
+      def self.parse(content)
+        ::MultiJson.load(content)
+      rescue ::MultiJson::ParseError => e
+        raise ::Karafka::Errors::ParserError, e
+      end
+
+      # @param content [Object] any object that we want to convert to a json string
+      # @return [String] Valid JSON string containing serialized data
+      # @raise [Karafka::Errors::ParserError] raised when we don't have a way to parse
+      #   given content to a json string format
+      # @note When string is passed to this method, we assume that it is already a json
+      #   string and we don't serialize it again. This allows us to serialize data before
+      #   it is being forwarded to a parser if we want to have a custom (not that simple)
+      #   json serialization
+      #
+      # @example From an ActiveRecord object
+      #   Json.generate(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
+      # @example From a string (no changes)
+      #   Json.generate("{\"a\":1}") #=> "{\"a\":1}"
+      def self.generate(content)
+        return content if content.is_a?(String)
+        return content.to_json if content.respond_to?(:to_json)
+        raise Karafka::Errors::ParserError, content
+      end
+    end
+  end
+end

data/lib/karafka/patches/dry_configurable.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for patches of external gems/libraries
+  module Patches
+    # Patch that will allow to use proc based lazy evaluated settings with Dry Configurable
+    # @see https://github.com/dry-rb/dry-configurable/blob/master/lib/dry/configurable.rb
+    module DryConfigurable
+      # We overwrite ::Dry::Configurable::Config to change on proc behaviour
+      # Unfortunately it does not provide an on call proc evaluation, so
+      # this feature had to be added here on demand/
+      # @param args Any arguments that DryConfigurable::Config accepts
+      def initialize(*args)
+        super
+
+        @config.each_key(&method(:rebuild))
+      end
+
+      private
+
+      # Method that rebuilds a given accessor, so when it consists a proc value, it will
+      # evaluate it upon return for blocks that don't require any arguments, otherwise
+      # it will return the block
+      # @param method_name [Symbol] name of an accessor that we want to rebuild
+      def rebuild(method_name)
+        define_singleton_method method_name do
+          value = super()
+          return value unless value.is_a?(Proc)
+          return value unless value.parameters.empty?
+          value.call
+        end
+      end
+    end
+  end
+end

data/lib/karafka/patches/ruby_kafka.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    # Patches for Ruby Kafka gem
+    module RubyKafka
+      # This patch allows us to inject business logic in between fetches and before the consumer
+      # stop, so we can perform stop commit or anything else that we need since
+      # ruby-kafka fetch loop does not allow that directly
+      # We don't wan't to use poll ruby-kafka api as it brings many more problems that we would
+      # have to take care of. That way, nothing like that ever happens but we get the control
+      # over the stopping process that we need (since we're the once that initiate it for each
+      # thread)
+      def consumer_loop
+        super do
+          consumers = Karafka::Persistence::Consumer
+                      .all
+                      .values
+                      .flat_map(&:values)
+                      .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
+
+          if Karafka::App.stopped?
+            consumers.each { |ctrl| ctrl.run_callbacks :before_stop }
+            Karafka::Persistence::Client.read.stop
+          else
+            consumers.each { |ctrl| ctrl.run_callbacks :before_poll }
+            yield
+            consumers.each { |ctrl| ctrl.run_callbacks :after_poll }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/client.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer client for further use
+    class Client
+      # Thread.current key under which we store current thread messages consumer client
+      PERSISTENCE_SCOPE = :client
+
+      # @param client [Karafka::Connection::Client] messages consumer client of
+      #   a current thread
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      def self.write(client)
+        Thread.current[PERSISTENCE_SCOPE] = client
+      end
+
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
+      #   client but we try to use it anyway
+      def self.read
+        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClient)
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
+  module Persistence
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
+    class Consumer
+      # Thread.current scope under which we store consumers data
+      PERSISTENCE_SCOPE = :consumers
+
+      class << self
+        # @return [Hash] current thread persistence scope hash with all the consumers
+        def all
+          # @note This does not need to be threadsafe (Hash) as it is always executed in a
+          # current thread context
+          Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+        end
+
+        # Used to build (if block given) and/or fetch a current consumer instance that will be
+        #   used to process messages from a given topic and partition
+        # @return [Karafka::BaseConsumer] base consumer descendant
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        def fetch(topic, partition)
+          # We always store a current instance for callback reasons
+          if topic.persistent
+            all[topic][partition] ||= topic.consumer.new
+          else
+            all[topic][partition] = topic.consumer.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/topic.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Local cache for routing topics
+    # We use it in order not to build string instances and remap incoming topic upon each
+    # message / message batches received
+    class Topic
+      # Thread.current scope under which we store topics data
+      PERSISTENCE_SCOPE = :topics
+
+      # @param group_id [String] group id for which we fetch a topic representation
+      # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
+      #   topic representation
+      # @return [Karafka::Routing::Topic] remapped topic representation that can be used further
+      #   on when working with given parameters
+      def self.fetch(group_id, raw_topic_name)
+        Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+
+        Thread.current[PERSISTENCE_SCOPE][group_id][raw_topic_name] ||= begin
+          # We map from incoming topic name, as it might be namespaced, etc.
+          # @see topic_mapper internal docs
+          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
+          Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to catch signals from ruby Signal class in order to manage Karafka stop
+  # @note There might be only one process - this class is a singleton
+  class Process
+    include Singleton
+
+    # Signal types that we handle
+    HANDLED_SIGNALS = %i[
+      SIGINT
+      SIGQUIT
+      SIGTERM
+    ].freeze
+
+    HANDLED_SIGNALS.each do |signal|
+      # Assigns a callback that will happen when certain signal will be send
+      # to Karafka server instance
+      # @note It does not define the callback itself -it needs to be passed in a block
+      # @example Define an action that should be taken on_sigint
+      #   process.on_sigint do
+      #     Karafka.logger.info('Log something here')
+      #     exit
+      #   end
+      define_method :"on_#{signal.to_s.downcase}" do |&block|
+        @callbacks[signal] << block
+      end
+    end
+
+    # Creates an instance of process and creates empty hash for callbacks
+    def initialize
+      @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
+    end
+
+    # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
+    # @note If there are no callbacks, this method will just ignore a given signal that was sent
+    # @yield [Block] block of code that we want to execute and supervise
+    def supervise
+      HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
+      yield
+    end
+
+    private
+
+    # Traps a single signal and performs callbacks (if any) or just ignores this signal
+    # @param [Symbol] signal type that we want to catch
+    def trap_signal(signal)
+      trap(signal) do
+        notice_signal(signal)
+        (@callbacks[signal] || []).each(&:call)
+      end
+    end
+
+    # Informs monitoring about trapped signal
+    # @param [Symbol] signal type that we received
+    # @note We cannot perform logging from trap context, that's why
+    #   we have to spin up a new thread to do this
+    def notice_signal(signal)
+      Thread.new do
+        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
+      end
+    end
+  end
+end

data/lib/karafka/responders/builder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Responders namespace encapsulates all the internal responder implementation parts
+  module Responders
+    # Responders builder is used to finding (based on the consumer class name) a responder
+    # that match the consumer. We use it when user does not provide a responder inside routing,
+    # but he still names responder with the same convention (and namespaces) as consumer
+    #
+    # @example Matching responder exists
+    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
+    # @example Matching responder does not exist
+    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
+    class Builder
+      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
+      #   Karafka::BaseConsumer
+      # @example Tries to find a responder that matches a given consumer. If nothing found,
+      #   will return nil (nil is accepted, because it means that a given consumer don't
+      #   pipe stuff further on)
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
+      end
+
+      # Tries to figure out a responder based on a consumer class name
+      # @return [Class] Responder class (not an instance)
+      # @return [nil] or nil if there's no matching responding class
+      def build
+        Helpers::ClassMatcher.new(
+          @consumer_class,
+          from: 'Consumer',
+          to: 'Responder'
+        ).match
+      end
+    end
+  end
+end

data/lib/karafka/responders/topic.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Responders
+    # Topic describes a single topic on which we want to respond with responding requirements
+    # @example Define topic (required by default)
+    #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
+    # @example Define optional topic
+    #   Karafka::Responders::Topic.new(:topic_name, required: false)
+    # @example Define topic that on which we want to respond multiple times
+    #   Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
+    class Topic
+      # Name of the topic on which we want to respond
+      attr_reader :name
+
+      # @param name [Symbol, String] name of a topic on which we want to respond
+      # @param options [Hash] non-default options for this topic
+      # @return [Karafka::Responders::Topic] topic description object
+      def initialize(name, options)
+        @name = name.to_s
+        @options = options
+      end
+
+      # @return [Boolean] is this a required topic (if not, it is optional)
+      def required?
+        @options.key?(:required) ? @options[:required] : true
+      end
+
+      # @return [Boolean] do we expect to use it multiple times in a single respond flow
+      def multiple_usage?
+        @options[:multiple_usage] || false
+      end
+
+      # @return [Boolean] was usage of this topic registered or not
+      def registered?
+        @options[:registered] == true
+      end
+
+      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
+      #   is safer and introduces less problems
+      def async?
+        @options.key?(:async) ? @options[:async] : false
+      end
+
+      # @return [Hash] hash with this topic attributes and options
+      def to_h
+        {
+          name: name,
+          multiple_usage: multiple_usage?,
+          required: required?,
+          registered: registered?,
+          async: async?
+        }
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Builder used as a DSL layer for building consumers and telling them which topics to consume
+    # @example Build a simple (most common) route
+    #   consumers do
+    #     topic :new_videos do
+    #       consumer NewVideosConsumer
+    #     end
+    #   end
+    class Builder < Array
+      include Singleton
+
+      # Used to draw routes for Karafka
+      # @note After it is done drawing it will store and validate all the routes to make sure that
+      #   they are correct and that there are no topic/group duplications (this is forbidden)
+      # @yield Evaluates provided block in a builder context so we can describe routes
+      # @example
+      #   draw do
+      #     topic :xyz do
+      #     end
+      #   end
+      def draw(&block)
+        instance_eval(&block)
+
+        each do |consumer_group|
+          hashed_group = consumer_group.to_h
+          validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
+          return if validation_result.success?
+          raise Errors::InvalidConfiguration, validation_result.errors
+        end
+      end
+
+      # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
+      #   we want to use. Since Karafka supports multi-process setup, we need to be able
+      #   to pick only those consumer groups that should be active in our given process context
+      def active
+        select(&:active?)
+      end
+
+      private
+
+      # Builds and saves given consumer group
+      # @param group_id [String, Symbol] name for consumer group
+      # @yield Evaluates a given block in a consumer group context
+      def consumer_group(group_id, &block)
+        consumer_group = ConsumerGroup.new(group_id.to_s)
+        self << Proxy.new(consumer_group, &block).target
+      end
+
+      # @param topic_name [String, Symbol] name of a topic from which we want to consumer
+      # @yield Evaluates a given block in a topic context
+      def topic(topic_name, &block)
+        consumer_group(topic_name) do
+          topic(topic_name, &block).tap(&:build)
+        end
+      end
+    end
+  end
+end