karafka 1.4.0 → 2.0.10

data/lib/karafka/params/builders/params.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Due to the fact, that we create params related objects in couple contexts / places
-    # plus backends can build up them their own way we have this namespace.
-    # It allows to isolate actual params objects from their building process that can be
-    # context dependent.
-    module Builders
-      # Builder for params
-      module Params
-        class << self
-          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
-          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
-          # @return [Karafka::Params::Params] params object with payload and message metadata
-          def from_kafka_message(kafka_message, topic)
-            metadata = Karafka::Params::Metadata.new(
-              create_time: kafka_message.create_time,
-              headers: kafka_message.headers || {},
-              is_control_record: kafka_message.is_control_record,
-              key: kafka_message.key,
-              offset: kafka_message.offset,
-              deserializer: topic.deserializer,
-              partition: kafka_message.partition,
-              receive_time: Time.now,
-              topic: topic.name
-            ).freeze
-
-            Karafka::Params::Params.new(
-              kafka_message.value,
-              metadata
-            )
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/params_batch.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating params batch instances
-      module ParamsBatch
-        class << self
-          # Creates params batch with params inside based on the incoming messages
-          # and the topic from which it comes
-          # @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
-          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
-          # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
-          def from_kafka_messages(kafka_messages, topic)
-            params_array = kafka_messages.map do |message|
-              Karafka::Params::Builders::Params.from_kafka_message(message, topic)
-            end
-
-            Karafka::Params::ParamsBatch.new(params_array).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/params_batch.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Params batch represents a set of messages received from Kafka.
-    # @note Params internally are lazy loaded before first use. That way we can skip
-    #   deserialization process if we have after_fetch that rejects some incoming messages
-    #   without using params It can be also used when handling really heavy data.
-    class ParamsBatch
-      include Enumerable
-
-      # @param params_array [Array<Karafka::Params::Params>] array with karafka params
-      # @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
-      def initialize(params_array)
-        @params_array = params_array
-      end
-
-      # @yieldparam [Karafka::Params::Params] each params instance
-      # @note Invocation of this method will not cause loading and deserializing each param after
-      #   another.
-      def each
-        @params_array.each { |param| yield(param) }
-      end
-
-      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
-      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
-      #   be deserialized
-      def deserialize!
-        each(&:payload)
-      end
-
-      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
-      #   we don't care about metadata and just want to extract all the data payloads from the
-      #   batch
-      def payloads
-        map(&:payload)
-      end
-
-      # @return [Karafka::Params::Params] first element
-      def first
-        @params_array.first
-      end
-
-      # @return [Karafka::Params::Params] last element
-      def last
-        @params_array.last
-      end
-
-      # @return [Integer] number of messages in the batch
-      def size
-        @params_array.size
-      end
-
-      # @return [Array<Karafka::Params::Params>] pure array with params
-      def to_a
-        @params_array
-      end
-    end
-  end
-end

data/lib/karafka/patches/ruby_kafka.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for various other libs patches
-  module Patches
-    # Patches for Ruby Kafka gem
-    module RubyKafka
-      # This patch allows us to inject business logic in between fetches and before the consumer
-      # stop, so we can perform stop commit or anything else that we need since
-      # ruby-kafka fetch loop does not allow that directly
-      # We don't won't to use poll ruby-kafka api as it brings many more problems that we would
-      # have to take care of. That way, nothing like that ever happens but we get the control
-      # over the stopping process that we need (since we're the once that initiate it for each
-      # thread)
-      def consumer_loop
-        super do
-          consumers = Karafka::Persistence::Consumers
-                      .current
-                      .values
-                      .flat_map(&:values)
-                      .select { |consumer| consumer.class.respond_to?(:after_fetch) }
-
-          if Karafka::App.stopping?
-            publish_event(consumers, 'before_stop')
-            Karafka::Persistence::Client.read.stop
-          else
-            publish_event(consumers, 'before_poll')
-            yield
-            publish_event(consumers, 'after_poll')
-          end
-        end
-      end
-
-      private
-
-      # Notifies consumers about particular events happening
-      # @param consumers [Array<Object>] all consumers that want to be notified about an event
-      # @param event_name [String] name of the event that happened
-      def publish_event(consumers, event_name)
-        consumers.each do |consumer|
-          key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
-          Karafka::App.monitor.instrument(key, context: consumer)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/client.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Persistence layer to store current thread messages consumer client for further use
-    class Client
-      # Thread.current key under which we store current thread messages consumer client
-      PERSISTENCE_SCOPE = :client
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @param client [Karafka::Connection::Client] messages consumer client of
-        #   a current thread
-        # @return [Karafka::Connection::Client] persisted messages consumer client
-        def write(client)
-          Thread.current[PERSISTENCE_SCOPE] = client
-        end
-
-        # @return [Karafka::Connection::Client] persisted messages consumer client
-        # @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
-        #   client but we try to use it anyway
-        def read
-          Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/consumers.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module used to provide a persistent cache layer for Karafka components that need to be
-  # shared inside of a same thread
-  module Persistence
-    # Module used to provide a persistent cache across batch requests for a given
-    # topic and partition to store some additional details when the persistent mode
-    # for a given topic is turned on
-    class Consumers
-      # Thread.current scope under which we store consumers data
-      PERSISTENCE_SCOPE = :consumers
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @return [Hash] current thread's persistence scope hash with all the consumers
-        def current
-          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
-            hash[key] = Concurrent::Hash.new
-          end
-        end
-
-        # Used to build (if block given) and/or fetch a current consumer instance that will be
-        #   used to process messages from a given topic and partition
-        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
-        # @param partition [Integer] number of partition for which we want to cache
-        # @return [Karafka::BaseConsumer] base consumer descendant
-        def fetch(topic, partition)
-          current[topic][partition] ||= topic.consumer.new(topic)
-        end
-
-        # Removes all persisted instances of consumers from the consumer cache
-        # @note This is used to reload consumers instances when code reloading in development mode
-        #   is present. This should not be used in production.
-        def clear
-          Thread
-            .list
-            .select { |thread| thread[PERSISTENCE_SCOPE] }
-            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/topics.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Local cache for routing topics
-    # We use it in order not to build string instances and remap incoming topic upon each
-    # message / message batches received
-    class Topics
-      # Thread.current scope under which we store topics data
-      PERSISTENCE_SCOPE = :topics
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @return [Concurrent::Hash] hash with all the topics from given groups
-        def current
-          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
-            hash[key] = Concurrent::Hash.new
-          end
-        end
-
-        # @param group_id [String] group id for which we fetch a topic representation
-        # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
-        #   topic representation
-        # @return [Karafka::Routing::Topics] remapped topic representation that can be used further
-        #   on when working with given parameters
-        def fetch(group_id, raw_topic_name)
-          current[group_id][raw_topic_name] ||= begin
-            # We map from incoming topic name, as it might be namespaced, etc.
-            # @see topic_mapper internal docs
-            mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
-            Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-          end
-        end
-
-        # Clears the whole topics cache for all the threads
-        # This is used for in-development code reloading as we need to get rid of all the
-        # preloaded and cached instances of objects to make it work
-        def clear
-          Thread
-            .list
-            .select { |thread| thread[PERSISTENCE_SCOPE] }
-            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/responders/builder.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Responders namespace encapsulates all the internal responder implementation parts
-  module Responders
-    # Responders builder is used for finding (based on the consumer class name) a responder
-    # that match the consumer. We use it when user does not provide a responder inside routing,
-    # but he still names responder with the same convention (and namespaces) as consumer
-    #
-    # @example Matching responder exists
-    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
-    # @example Matching responder does not exist
-    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
-    class Builder
-      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
-      #   Karafka::BaseConsumer
-      # @example Tries to find a responder that matches a given consumer. If nothing found,
-      #   will return nil (nil is accepted, because it means that a given consumer don't
-      #   pipe stuff further on)
-      def initialize(consumer_class)
-        @consumer_class = consumer_class
-      end
-
-      # Tries to figure out a responder based on a consumer class name
-      # @return [Class] Responder class (not an instance)
-      # @return [nil] or nil if there's no matching responding class
-      def build
-        Helpers::ClassMatcher.new(
-          @consumer_class,
-          from: 'Consumer',
-          to: 'Responder'
-        ).match
-      end
-    end
-  end
-end

data/lib/karafka/responders/topic.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Responders
-    # Topic describes a single topic on which we want to respond with responding requirements
-    # @example Define topic (required by default)
-    #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
-    # @example Define optional topic
-    #   Karafka::Responders::Topic.new(:topic_name, required: false)
-    class Topic
-      # Name of the topic on which we want to respond
-      attr_reader :name
-
-      # @param name [Symbol, String] name of a topic on which we want to respond
-      # @param options [Hash] non-default options for this topic
-      # @return [Karafka::Responders::Topic] topic description object
-      def initialize(name, options)
-        @name = name.to_s
-        @options = options
-      end
-
-      # @return [Boolean] is this a required topic (if not, it is optional)
-      def required?
-        @options.key?(:required) ? @options[:required] : true
-      end
-
-      # @return [Boolean] was usage of this topic registered or not
-      def registered?
-        @options[:registered] == true
-      end
-
-      # @return [Class] Class to use to serialize messages for this topic
-      def serializer
-        @options[:serializer]
-      end
-
-      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
-      #   is safer and introduces less problems
-      def async?
-        @options.key?(:async) ? @options[:async] : false
-      end
-
-      # @return [Hash] hash with this topic attributes and options
-      def to_h
-        {
-          name: name,
-          required: required?,
-          registered: registered?,
-          serializer: serializer,
-          async: async?
-        }
-      end
-    end
-  end
-end

data/lib/karafka/routing/topic_mapper.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    # Default topic mapper that does not remap things
-    # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
-    # provider dependent, we can then define mapper and use internally "pure" topic names in
-    # routes and responders
-    #
-    # @example Mapper for mapping prefixed topics
-    #   class MyMapper
-    #     PREFIX = "my_user_name."
-    #
-    #     def incoming(topic)
-    #       topic.to_s.gsub(PREFIX, '')
-    #     end
-    #
-    #     def outgoing(topic)
-    #       "#{PREFIX}#{topic}"
-    #     end
-    #   end
-    #
-    # @example Mapper for replacing "." with "_" in topic names
-    #   class MyMapper
-    #     PREFIX = "my_user_name."
-    #
-    #     def incoming(topic)
-    #       topic.to_s.gsub('.', '_')
-    #     end
-    #
-    #     def outgoing(topic)
-    #       topic.to_s.gsub('_', '.')
-    #     end
-    #   end
-    class TopicMapper
-      # @param topic [String, Symbol] topic
-      # @return [String, Symbol] same topic as on input
-      # @example
-      #   incoming('topic_name') #=> 'topic_name'
-      def incoming(topic)
-        topic
-      end
-
-      # @param topic [String, Symbol] topic
-      # @return [String, Symbol] same topic as on input
-      # @example
-      #   outgoing('topic_name') #=> 'topic_name'
-      def outgoing(topic)
-        topic
-      end
-    end
-  end
-end

data/lib/karafka/serialization/json/serializer.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module for all supported by default serialization and deserialization ways
-  module Serialization
-    module Json
-      # Default Karafka Json serializer for serializing data
-      class Serializer
-        # @param content [Object] any object that we want to convert to a json string
-        # @return [String] Valid JSON string containing serialized data
-        # @raise [Karafka::Errors::SerializationError] raised when we don't have a way to
-        #   serialize provided data to json
-        # @note When string is passed to this method, we assume that it is already a json
-        #   string and we don't serialize it again. This allows us to serialize data before
-        #   it is being forwarded to this serializer if we want to have a custom (not that simple)
-        #   json serialization
-        #
-        # @example From an ActiveRecord object
-        #   Serializer.call(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
-        # @example From a string (no changes)
-        #   Serializer.call("{\"a\":1}") #=> "{\"a\":1}"
-        def call(content)
-          return content if content.is_a?(String)
-          return content.to_json if content.respond_to?(:to_json)
-
-          raise Karafka::Errors::SerializationError, content
-        end
-      end
-    end
-  end
-end

data/lib/karafka/setup/configurators/water_drop.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Setup
-    # Configurators are used to post setup some of the components of Karafka after the core
-    # framework is initialized
-    module Configurators
-      # Class responsible for setting up WaterDrop configuration
-      class WaterDrop
-        # Sets up a WaterDrop settings
-        # @param config [Karafka::Setup::Config] Config we can user to setup things
-        # @note This will also inject Karafka monitor as a default monitor into WaterDrop,
-        #   so we have the same monitor within whole Karafka framework (same with logger)
-        def call(config)
-          ::WaterDrop.setup do |water_config|
-            water_config.deliver = true
-
-            config.to_h.reject { |k, _v| k == :kafka }.each do |k, v|
-              key_assignment = :"#{k}="
-              next unless water_config.respond_to?(key_assignment)
-
-              water_config.public_send(key_assignment, v)
-            end
-
-            config.kafka.to_h.each do |k, v|
-              key_assignment = :"#{k}="
-              next unless water_config.kafka.respond_to?(key_assignment)
-
-              water_config.kafka.public_send(key_assignment, v)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/templates/application_responder.rb.erb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-# Application responder from which all Karafka responders should inherit
-# You can rename it if it would conflict with your current code base (in case you're integrating
-# Karafka with other frameworks)
-class ApplicationResponder < Karafka::BaseResponder
-  # This method needs to be implemented in each of the responders
-  # def respond(data)
-  #   respond_to :topic, data.to_json
-  # end
-end