karafka 1.3.3 → 1.4.0.pre.rc1

data/lib/karafka/attributes_map.rb

@@ -52,14 +52,8 @@ module Karafka
         ignored_settings = api_adapter[:subscribe]
         defined_settings = api_adapter.values.flatten
         karafka_settings = %i[batch_fetching]
-        # This is a dirty and bad hack of dry-configurable to get keys before setting values
-        dynamically_proxied = Karafka::Setup::Config
-                              ._settings
-                              .settings
-                              .find { |s| s.name == :kafka }
-                              .value
-                              .names
-                              .to_a
+
+        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
 
         (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
       end

data/lib/karafka/cli.rb

@@ -47,7 +47,7 @@ end
 if ENV['KARAFKA_CONSOLE']
   # Reloads Karafka irb console session
   def reload!
-    puts "Reloading...\n"
+    Karafka.logger.info "Reloading...\n"
     Kernel.exec Karafka::Cli::Console.command
   end
 end

data/lib/karafka/cli/flow.rb

@@ -11,19 +11,22 @@ module Karafka
       def call
         topics.each do |topic|
           any_topics = !topic.responder&.topics.nil?
+          log_messages = []
 
           if any_topics
-            puts "#{topic.name} =>"
+            log_messages << "#{topic.name} =>"
 
             topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
 
-              print responder_topic.name, "(#{features.join(', ')})"
+              log_messages << format(responder_topic.name, "(#{features.join(', ')})")
             end
           else
-            puts "#{topic.name} => (nothing)"
+            log_messages << "#{topic.name} => (nothing)"
           end
+
+          Karafka.logger.info(log_messages.join("\n"))
         end
       end
 
@@ -34,11 +37,11 @@ module Karafka
         Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
       end
 
-      # Prints a given value with label in a nice way
+      # Formats a given value with label in a nice way
       # @param label [String] label describing value
       # @param value [String] value that should be printed
-      def print(label, value)
-        printf "%-25s %s\n", "  - #{label}:", value
+      def format(label, value)
+        "  - #{label}:                #{value}"
       end
     end
   end

data/lib/karafka/cli/info.rb

@@ -24,7 +24,7 @@ module Karafka
           "Kafka seed brokers: #{config.kafka.seed_brokers}"
         ]
 
-        puts(info.join("\n"))
+        Karafka.logger.info(info.join("\n"))
       end
     end
   end

data/lib/karafka/connection/api_adapter.rb

@@ -14,11 +14,12 @@ module Karafka
     module ApiAdapter
       class << self
         # Builds all the configuration settings for Kafka.new method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
         # @return [Array<Hash>] Array with all the client arguments including hash with all
         #   the settings required by Kafka.new method
         # @note We return array, so we can inject any arguments we want, in case of changes in the
         #   raw driver
-        def client
+        def client(consumer_group)
           # This one is a default that takes all the settings except special
           # cases defined in the map
           settings = {
@@ -26,14 +27,17 @@ module Karafka
             client_id: ::Karafka::App.config.client_id
           }
 
-          kafka_configs.each do |setting_name, setting_value|
+          kafka_configs.each_key do |setting_name|
             # All options for config adapter should be ignored as we're just interested
             # in what is left, as we want to pass all the options that are "typical"
             # and not listed in the api_adapter special cases mapping. All the values
             # from the api_adapter mapping go somewhere else, not to the client directly
             next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
 
-            settings[setting_name] = setting_value
+            # Settings for each consumer group are either defined per consumer group or are
+            # inherited from the global/general settings level, thus we don't have to fetch them
+            # from the kafka settings as they are already on a consumer group level
+            settings[setting_name] = consumer_group.public_send(setting_name)
           end
 
           settings_hash = sanitize(settings)
@@ -105,11 +109,13 @@ module Karafka
           # Majority of users don't use custom topic mappers. No need to change anything when it
           # is a default mapper that does not change anything. Only some cloud providers require
           # topics to be remapped
-          return [params] if Karafka::App.config.topic_mapper.is_a?(Karafka::Routing::TopicMapper)
+          return [params.metadata] if Karafka::App.config.topic_mapper.is_a?(
+            Karafka::Routing::TopicMapper
+          )
 
           # @note We don't use tap as it is around 13% slower than non-dup version
-          dupped = params.dup
-          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
+          dupped = params.metadata.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.metadata.topic)
           [dupped]
         end
 

data/lib/karafka/connection/batch_delegator.rb

@@ -23,7 +23,11 @@ module Karafka
           ) do
             # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
             # level only available for batch consuming
-            consumer.metadata = Params::Builders::Metadata.from_kafka_batch(kafka_batch, topic)
+            consumer.batch_metadata = Params::Builders::BatchMetadata.from_kafka_batch(
+              kafka_batch,
+              topic
+            )
+
             kafka_messages = kafka_batch.messages
 
             # Depending on a case (persisted or not) we might use new consumer instance per

data/lib/karafka/connection/builder.rb

@@ -6,9 +6,11 @@ module Karafka
     module Builder
       class << self
         # Builds a Kafka::Client instance that we use to work with Kafka cluster
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which we want
+        #   to have a new Kafka client
         # @return [::Kafka::Client] returns a Kafka client
-        def call
-          Kafka.new(*ApiAdapter.client)
+        def call(consumer_group)
+          Kafka.new(*ApiAdapter.client(consumer_group))
         end
       end
     end

data/lib/karafka/connection/client.rb

@@ -97,7 +97,7 @@ module Karafka
       def kafka_consumer
         # @note We don't cache the connection internally because we cache kafka_consumer that uses
         #   kafka client object instance
-        @kafka_consumer ||= Builder.call.consumer(
+        @kafka_consumer ||= Builder.call(consumer_group).consumer(
           *ApiAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|

data/lib/karafka/consumers/batch_metadata.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Brings the batch metadata into consumers that support batch_fetching
+    module BatchMetadata
+      attr_accessor :batch_metadata
+    end
+  end
+end

data/lib/karafka/consumers/includer.rb

@@ -16,7 +16,7 @@ module Karafka
 
           bind_backend(consumer, topic)
           bind_params(consumer, topic)
-          bind_metadata(consumer, topic)
+          bind_batch_metadata(consumer, topic)
           bind_responders(consumer, topic)
         end
 
@@ -40,13 +40,14 @@ module Karafka
           consumer.extend(SingleParams)
         end
 
-        # Adds an option to work with metadata for consumer instances that have batch fetching
+        # Adds an option to work with batch metadata for consumer instances that have
+        #   batch fetching enabled
         # @param consumer [Karafka::BaseConsumer] consumer instance
         # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_metadata(consumer, topic)
+        def bind_batch_metadata(consumer, topic)
           return unless topic.batch_fetching
 
-          consumer.extend(Metadata)
+          consumer.extend(BatchMetadata)
         end
 
         # Adds responders support for topics and consumers with responders defined for them

data/lib/karafka/contracts/server_cli_options.rb

@@ -6,6 +6,8 @@ module Karafka
     # We validate some basics + the list of consumer_groups on which we want to use, to make
     # sure that all of them are defined, plus that a pidfile does not exist
     class ServerCliOptions < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
       params do
         optional(:pid).filled(:str?)
         optional(:daemon).filled(:bool?)

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -43,7 +43,7 @@ module Karafka
         # so it returns a topic as a string, not a routing topic
         debug(
           <<~MSG.chomp.tr("\n", ' ')
-            Params deserialization for #{event[:caller].topic} topic
+            Params deserialization for #{event[:caller].metadata.topic} topic
             successful in #{event[:time]} ms
           MSG
         )
@@ -52,7 +52,9 @@ module Karafka
       # Logs unsuccessful deserialization attempts of incoming data
       # @param event [Dry::Events::Event] event details including payload
       def on_params_params_deserialize_error(event)
-        error "Params deserialization error for #{event[:caller].topic} topic: #{event[:error]}"
+        topic = event[:caller].metadata.topic
+        error = event[:error]
+        error "Params deserialization error for #{topic} topic: #{error}"
       end
 
       # Logs errors that occurred in a listener fetch loop

data/lib/karafka/params/batch_metadata.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Simple batch metadata object that stores all non-message information received from Kafka
+    # cluster while fetching the data
+    # @note This metadata object refers to per batch metadata, not `#params.metadata`
+    BatchMetadata = Struct.new(
+      :batch_size,
+      :first_offset,
+      :highwater_mark_offset,
+      :unknown_last_offset,
+      :last_offset,
+      :offset_lag,
+      :deserializer,
+      :partition,
+      :topic,
+      keyword_init: true
+    ) do
+      # @return [Boolean] is the last offset known or unknown
+      def unknown_last_offset?
+        unknown_last_offset
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/batch_metadata.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data
+          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @return [Karafka::Params::BatchMetadata] batch metadata object
+          def from_kafka_batch(kafka_batch, topic)
+            Karafka::Params::BatchMetadata.new(
+              batch_size: kafka_batch.messages.count,
+              first_offset: kafka_batch.first_offset,
+              highwater_mark_offset: kafka_batch.highwater_mark_offset,
+              unknown_last_offset: kafka_batch.unknown_last_offset?,
+              last_offset: kafka_batch.last_offset,
+              offset_lag: kafka_batch.offset_lag,
+              deserializer: topic.deserializer,
+              partition: kafka_batch.partition,
+              topic: topic.name
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params.rb

@@ -12,22 +12,24 @@ module Karafka
         class << self
           # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
           # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
-          # @return [Karafka::Params::Params] params object
+          # @return [Karafka::Params::Params] params object with payload and message metadata
           def from_kafka_message(kafka_message, topic)
-            Karafka::Params::Params
-              .new
-              .merge!(
-                'create_time' => kafka_message.create_time,
-                'headers' => kafka_message.headers || {},
-                'is_control_record' => kafka_message.is_control_record,
-                'key' => kafka_message.key,
-                'offset' => kafka_message.offset,
-                'deserializer' => topic.deserializer,
-                'partition' => kafka_message.partition,
-                'receive_time' => Time.now,
-                'topic' => kafka_message.topic,
-                'payload' => kafka_message.value
-              )
+            metadata = Karafka::Params::Metadata.new(
+              create_time: kafka_message.create_time,
+              headers: kafka_message.headers || {},
+              is_control_record: kafka_message.is_control_record,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              receive_time: Time.now,
+              topic: topic.name
+            ).freeze
+
+            Karafka::Params::Params.new(
+              kafka_message.value,
+              metadata
+            )
           end
         end
       end

data/lib/karafka/params/builders/params_batch.rb

@@ -12,11 +12,11 @@ module Karafka
           # @param topic [Karafka::Routing::Topic] topic for which we're received messages
           # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
           def from_kafka_messages(kafka_messages, topic)
-            params_array = kafka_messages.map! do |message|
+            params_array = kafka_messages.map do |message|
               Karafka::Params::Builders::Params.from_kafka_message(message, topic)
             end
 
-            Karafka::Params::ParamsBatch.new(params_array)
+            Karafka::Params::ParamsBatch.new(params_array).freeze
           end
         end
       end

data/lib/karafka/params/metadata.rb

@@ -2,34 +2,19 @@
 
 module Karafka
   module Params
-    # Simple metadata object that stores all non-message information received from Kafka cluster
-    # while fetching the data
-    class Metadata < Hash
-      # Attributes that should be accessible as methods as well (not only hash)
-      METHOD_ATTRIBUTES = %w[
-        batch_size
-        first_offset
-        highwater_mark_offset
-        last_offset
-        offset_lag
-        deserializer
-        partition
-        topic
-      ].freeze
-
-      private_constant :METHOD_ATTRIBUTES
-
-      METHOD_ATTRIBUTES.each do |attr|
-        # Defines a method call accessor to a particular hash field.
-        define_method(attr) do
-          self[attr]
-        end
-      end
-
-      # @return [Boolean] is the last offset known or unknown
-      def unknown_last_offset?
-        self['unknown_last_offset']
-      end
-    end
+    # Single message / params metadata details that can be accessed without the need for the
+    # payload deserialization
+    Metadata = Struct.new(
+      :create_time,
+      :headers,
+      :is_control_record,
+      :key,
+      :offset,
+      :deserializer,
+      :partition,
+      :receive_time,
+      :topic,
+      keyword_init: true
+    )
   end
 end

data/lib/karafka/params/params.rb

@@ -6,58 +6,40 @@ module Karafka
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using deserializer until we execute our logic. That way we can operate with
     # heavy-deserialization data without slowing down the whole application.
-    class Params < Hash
-      # Params attributes that should be available via a method call invocation for Kafka
-      # client compatibility.
-      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
-      # uses those fields via method calls, so in order to be able to pass there our params
-      # objects, have to have same api.
-      METHOD_ATTRIBUTES = %w[
-        create_time
-        headers
-        is_control_record
-        key
-        offset
-        deserializer
-        deserialized
-        partition
-        receive_time
-        topic
-        payload
-      ].freeze
+    class Params
+      attr_reader :raw_payload, :metadata
 
-      private_constant :METHOD_ATTRIBUTES
-
-      METHOD_ATTRIBUTES.each do |attr|
-        # Defines a method call accessor to a particular hash field.
-        # @note Won't work for complex key names that contain spaces, etc
-        # @param key [Symbol] name of a field that we want to retrieve with a method call
-        # @example
-        #   key_attr_reader :example
-        #   params.example #=> 'my example payload'
-        define_method(attr) do
-          self[attr]
-        end
+      # @param raw_payload [Object] incoming payload before deserialization
+      # @param metadata [Karafka::Params::Metadata] message metadata object
+      def initialize(raw_payload, metadata)
+        @raw_payload = raw_payload
+        @metadata = metadata
+        @deserialized = false
+        @payload = nil
       end
 
-      # @return [Karafka::Params::Params] This method will trigger deserializer execution. If we
-      #   decide to retrieve data, deserializer will be executed to get data. Output of that will
-      #   be merged to the current object. This object will be also marked as already deserialized,
-      #   so we won't deserialize it again.
-      def deserialize!
-        return self if self['deserialized']
+      # @return [Object] lazy-deserialized data (deserialized upon first request)
+      def payload
+        return @payload if deserialized?
+
+        @payload = deserialize
+        # We mark deserialization as successful after deserialization, as in case of an error
+        # this won't be falsely set to true
+        @deserialized = true
+        @payload
+      end
 
-        self['deserialized'] = true
-        self['payload'] = deserialize
-        self
+      # @return [Boolean] did given params payload were deserialized already
+      def deserialized?
+        @deserialized
       end
 
       private
 
-      # @return [Object] deserialized data
+      # @return [Object] tries de-serializes data
       def deserialize
         Karafka.monitor.instrument('params.params.deserialize', caller: self) do
-          self['deserializer'].call(self)
+          metadata.deserializer.call(self)
         end
       rescue ::StandardError => e
         Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)