karafka 1.4.9 → 2.0.0.alpha1

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -3,7 +3,7 @@
 module Karafka
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
-    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    # It can be removed/replaced or anything without any harm to the Karafka app flow.
     class StdoutListener
       # Log levels that we use in this particular listener
       USED_LOG_LEVELS = %i[
@@ -13,121 +13,94 @@ module Karafka
         fatal
       ].freeze
 
-      # Logs details about incoming batches and with which consumer we will consume them
-      # @param event [Dry::Events::Event] event details including payload
-      def on_connection_batch_delegator_call(event)
-        consumer = event[:consumer]
-        topic = consumer.topic.name
-        kafka_messages = event[:kafka_batch].messages
-        info(
-          <<~MSG.chomp.tr("\n", ' ')
-            #{kafka_messages.count} messages
-            on #{topic} topic
-            delegated to #{consumer.class}
-          MSG
-        )
-      end
-
-      # Logs details about incoming message and with which consumer we will consume it
-      # @param event [Dry::Events::Event] event details including payload
-      def on_connection_message_delegator_call(event)
-        consumer = event[:consumer]
-        topic = consumer.topic.name
-        info "1 message on #{topic} topic delegated to #{consumer.class}"
-      end
-
-      # Logs details about each received message value deserialization
-      # @param event [Dry::Events::Event] event details including payload
-      def on_params_params_deserialize(event)
-        # Keep in mind, that a caller here is a param object not a controller,
-        # so it returns a topic as a string, not a routing topic
-        debug(
-          <<~MSG.chomp.tr("\n", ' ')
-            Params deserialization for #{event[:caller].metadata.topic} topic
-            successful in #{event[:time]} ms
-          MSG
-        )
-      end
-
-      # Logs unsuccessful deserialization attempts of incoming data
-      # @param event [Dry::Events::Event] event details including payload
-      def on_params_params_deserialize_error(event)
-        topic = event[:caller].metadata.topic
-        error = event[:error]
-        error "Params deserialization error for #{topic} topic: #{error}"
-      end
-
-      # Logs errors that occurred in a listener fetch loop
-      # @param event [Dry::Events::Event] event details including payload
-      # @note It's an error as we can recover from it not a fatal
-      def on_connection_listener_fetch_loop_error(event)
-        error "Listener fetch loop error: #{event[:error]}"
-      end
-
-      # Logs errors that are related to the connection itself
-      # @param event [Dry::Events::Event] event details including payload
-      # @note Karafka will attempt to reconnect, so an error not a fatal
-      def on_connection_client_fetch_loop_error(event)
-        error "Client fetch loop error: #{event[:error]}"
-      end
-
-      # Logs info about crashed fetcher
-      # @param event [Dry::Events::Event] event details including payload
-      # @note If this happens, Karafka will shutdown as it means a critical error
-      #   in one of the threads
-      def on_fetcher_call_error(event)
-        fatal "Fetcher crash due to an error: #{event[:error]}"
+      # Logs each messages fetching attempt
+      #
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_connection_listener_fetch_loop(_event)
+        info 'Receiving new messages from Kafka...'
       end
 
-      # Logs info about processing of a certain dataset with an inline backend
+      # Logs about messages that we've received from Kafka
+      #
       # @param event [Dry::Events::Event] event details including payload
-      def on_backends_inline_process(event)
-        count = event[:caller].send(:params_batch).to_a.size
-        topic = event[:caller].topic.name
-        time = event[:time]
-        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+      def on_connection_listener_fetch_loop_received(event)
+        info "Received #{event[:messages_buffer].size} new messages from Kafka"
       end
 
-      # Logs info about system signals that Karafka received
+      # Logs info about system signals that Karafka received.
+      #
       # @param event [Dry::Events::Event] event details including payload
       def on_process_notice_signal(event)
         info "Received #{event[:signal]} system signal"
       end
 
-      # Logs info about responder usage withing a controller flow
-      # @param event [Dry::Events::Event] event details including payload
-      def on_consumers_responders_respond_with(event)
-        calling = event[:caller]
-        responder = calling.topic.responder
-        data = event[:data]
-        info "Responded from #{calling.class} using #{responder} with following data #{data}"
-      end
-
-      # Logs info that we're initializing Karafka framework components
+      # Logs info that we're initializing Karafka app.
+      #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_initializing(_event)
-        info "Initializing Karafka framework #{::Process.pid}"
+        info 'Initializing Karafka framework'
       end
 
-      # Logs info that we're running Karafka app
+      # Logs info that we're running Karafka app.
+      #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_running(_event)
-        info "Running Karafka server #{::Process.pid}"
+        info 'Running Karafka server'
       end
 
-      # Logs info that we're going to stop the Karafka server
+      # Logs info that we're going to stop the Karafka server.
+      #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_stopping(_event)
         # We use a separate thread as logging can't be called from trap context
-        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+        Thread.new { info 'Stopping Karafka server' }
       end
 
-      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
-      #   forced exit
+      # Logs info that we stopped the Karafka server.
+      #
       # @param _event [Dry::Events::Event] event details including payload
-      def on_app_stopping_error(_event)
+      def on_app_stopped(_event)
         # We use a separate thread as logging can't be called from trap context
-        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+        Thread.new { info 'Stopped Karafka server' }
+      end
+
+      # There are many types of errors that can occur in many places, but we provide a single
+      # handler for all of them to simplify error instrumentation.
+      # @param event [Dry::Events::Event] event details including payload
+      def on_error_occurred(event)
+        type = event[:type]
+        error = event[:error]
+        details = (error.backtrace || []).join("\n")
+
+        case type
+        when 'consumer.consume.error'
+          error "Consumer consuming error: #{error}"
+          error details
+        when 'consumer.revoked.error'
+          error "Consumer on revoked failed due to an error: #{error}"
+          error details
+        when 'consumer.shutdown.error'
+          error "Consumer on shutdown failed due to an error: #{error}"
+          error details
+        when 'worker.process.error'
+          fatal "Worker processing failed due to an error: #{error}"
+          fatal details
+        when 'connection.listener.fetch_loop.error'
+          error "Listener fetch loop error: #{error}"
+          error details
+        when 'runner.call.error'
+          fatal "Runner crashed due to an error: #{error}"
+          fatal details
+        when 'app.stopping.error'
+          # We use a separate thread as logging can't be called from trap context
+          Thread.new { error 'Forceful Karafka server stop' }
+        when 'librdkafka.error'
+          error "librdkafka internal error occurred: #{error}"
+          error details
+        else
+          # This should never happen. Please contact the maintainers
+          raise Errors::UnsupportedCaseError, event
+        end
       end
 
       USED_LOG_LEVELS.each do |log_level|

data/lib/karafka/instrumentation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  # @note Since we can only have one statistics callbacks manager and one error callbacks manager
+  # we use WaterDrops one that is already configured.
+  module Instrumentation
+    class << self
+      # Returns a manager for statistics callbacks
+      # @return [::WaterDrop::CallbacksManager]
+      def statistics_callbacks
+        ::WaterDrop::Instrumentation.statistics_callbacks
+      end
+
+      # Returns a manager for error callbacks
+      # @return [::WaterDrop::CallbacksManager]
+      def error_callbacks
+        ::WaterDrop::Instrumentation.error_callbacks
+      end
+    end
+  end
+end

data/lib/karafka/licenser.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Checks the license presence for pro and loads pro components when needed (if any)
+  class Licenser
+    # Location in the gem where we store the public key
+    PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, 'certs', 'karafka-pro.pem')
+
+    private_constant :PUBLIC_KEY_LOCATION
+
+    # Check license and setup license details (if needed)
+    # @param license_config [Dry::Configurable::Config] config part related to the licensing
+    def verify(license_config)
+      # If no license, it will just run LGPL components without anything extra
+      return unless license_config.token
+
+      public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
+
+      # We gsub and strip in case someone copy-pasted it as a multi line string
+      formatted_token = license_config.token.strip.delete("\n").delete(' ')
+      decoded_token = Base64.decode64(formatted_token)
+
+      begin
+        data = public_key.public_decrypt(decoded_token)
+      rescue OpenSSL::OpenSSLError
+        data = nil
+      end
+
+      details = data ? JSON.parse(data) : raise_invalid_license_token
+
+      license_config.entity = details.fetch('entity')
+      license_config.expires_on = Date.parse(details.fetch('expires_on'))
+
+      return if license_config.expires_on > Date.today
+
+      notify_if_license_expired(license_config.expires_on)
+    end
+
+    private
+
+    # Raises an error with info, that used token is invalid
+    def raise_invalid_license_token
+      raise(
+        Errors::InvalidLicenseTokenError,
+        <<~MSG.tr("\n", ' ')
+          License key you provided is invalid.
+          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+        MSG
+      )
+    end
+
+    # We do not raise an error here as we don't want to cause any problems to someone that runs
+    # Karafka on production. Error is enough.
+    # @param expires_on [Date] when the license expires
+    def notify_if_license_expired(expires_on)
+      Karafka.logger.error(
+        <<~MSG.tr("\n", ' ')
+          Your license expired on #{expires_on}.
+          Karafka no longer uses any Pro capabilities.
+          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+        MSG
+      )
+    end
+  end
+end

data/lib/karafka/{params → messages}/batch_metadata.rb

@@ -1,26 +1,20 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Params
+  module Messages
     # Simple batch metadata object that stores all non-message information received from Kafka
-    # cluster while fetching the data
-    # @note This metadata object refers to per batch metadata, not `#params.metadata`
+    # cluster while fetching the data.
+    #
+    # @note This metadata object refers to per batch metadata, not `#message.metadata`
     BatchMetadata = Struct.new(
-      :batch_size,
+      :size,
       :first_offset,
-      :highwater_mark_offset,
-      :unknown_last_offset,
       :last_offset,
-      :offset_lag,
       :deserializer,
       :partition,
       :topic,
+      :scheduled_at,
       keyword_init: true
-    ) do
-      # @return [Boolean] is the last offset known or unknown
-      def unknown_last_offset?
-        unknown_last_offset
-      end
-    end
+    )
   end
 end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations.
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data.
+          #
+          # @param kafka_batch [Array<Rdkafka::Consumer::Message>] raw fetched messages
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @param scheduled_at [Time] moment when the batch was scheduled for processing
+          # @return [Karafka::Messages::BatchMetadata] batch metadata object
+          def call(kafka_batch, topic, scheduled_at)
+            Karafka::Messages::BatchMetadata.new(
+              size: kafka_batch.count,
+              first_offset: kafka_batch.first.offset,
+              last_offset: kafka_batch.last.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_batch[0].partition,
+              topic: topic.name,
+              scheduled_at: scheduled_at
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/message.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Builders encapsulate logic related to creating messages related objects.
+    module Builders
+      # Builder of a single message based on raw rdkafka message.
+      module Message
+        class << self
+          # @param kafka_message [Rdkafka::Consumer::Message] raw fetched message
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @param received_at [Time] moment when we've received the message
+          # @return [Karafka::Messages::Message] message object with payload and metadata
+          def call(kafka_message, topic, received_at)
+            # @see https://github.com/appsignal/rdkafka-ruby/issues/168
+            kafka_message.headers.transform_keys!(&:to_s)
+
+            metadata = Karafka::Messages::Metadata.new(
+              timestamp: kafka_message.timestamp,
+              headers: kafka_message.headers,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              topic: topic.name,
+              received_at: received_at
+            ).freeze
+
+            Karafka::Messages::Message.new(
+              kafka_message.payload,
+              metadata
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/messages.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    module Builders
+      # Builder for creating message batch instances.
+      module Messages
+        class << self
+          # Creates messages batch with messages inside based on the incoming messages and the
+          # topic from which it comes.
+          #
+          # @param kafka_messages [Array<Rdkafka::Consumer::Message>] raw fetched messages
+          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @param received_at [Time] moment in time when the messages were received
+          # @return [Karafka::Messages::Messages] messages batch object
+          def call(kafka_messages, topic, received_at)
+            messages_array = kafka_messages.map do |message|
+              Karafka::Messages::Builders::Message.call(
+                message,
+                topic,
+                received_at
+              )
+            end
+
+            metadata = BatchMetadata.call(
+              kafka_messages,
+              topic,
+              received_at
+            ).freeze
+
+            Karafka::Messages::Messages.new(
+              messages_array,
+              metadata
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/{params/params.rb → messages/message.rb}

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Params namespace encapsulating all the logic that is directly related to params handling
-  module Params
+  # Messages namespace encapsulating all the logic that is directly related to messages handling
+  module Messages
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using deserializer until we execute our logic. That way we can operate with
     # heavy-deserialization data without slowing down the whole application.
-    class Params
+    class Message
       extend Forwardable
 
       attr_reader :raw_payload, :metadata
@@ -14,7 +14,7 @@ module Karafka
       def_delegators :metadata, *Metadata.members
 
       # @param raw_payload [Object] incoming payload before deserialization
-      # @param metadata [Karafka::Params::Metadata] message metadata object
+      # @param metadata [Karafka::Messages::Metadata] message metadata object
       def initialize(raw_payload, metadata)
         @raw_payload = raw_payload
         @metadata = metadata
@@ -33,21 +33,16 @@ module Karafka
         @payload
       end
 
-      # @return [Boolean] did given params payload were deserialized already
+      # @return [Boolean] did we deserialize payload already
       def deserialized?
         @deserialized
       end
 
       private
 
-      # @return [Object] tries de-serializes data
+      # @return [Object] deserialized data
       def deserialize
-        Karafka.monitor.instrument('params.params.deserialize', caller: self) do
-          metadata.deserializer.call(self)
-        end
-      rescue ::StandardError => e
-        Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
-        raise e
+        metadata.deserializer.call(self)
       end
     end
   end

data/lib/karafka/messages/messages.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Messages batch represents a set of messages received from Kafka of a single topic partition.
+    class Messages
+      include Enumerable
+
+      attr_reader :metadata
+
+      # @param messages_array [Array<Karafka::Messages::Message>] array with karafka messages
+      # @param metadata [Karafka::Messages::BatchMetadata]
+      # @return [Karafka::Messages::Messages] lazy evaluated messages batch object
+      def initialize(messages_array, metadata)
+        @messages_array = messages_array
+        @metadata = metadata
+      end
+
+      # @param block [Proc] block we want to execute per each message
+      # @note Invocation of this method will not cause loading and deserializing of messages.
+      def each(&block)
+        @messages_array.each(&block)
+      end
+
+      # Runs deserialization of all the messages and returns them
+      # @return [Array<Karafka::Messages::Message>]
+      def deserialize!
+        each(&:payload)
+      end
+
+      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
+      #   we don't care about metadata and just want to extract all the data payloads from the
+      #   batch
+      def payloads
+        map(&:payload)
+      end
+
+      # @return [Array<String>] array with raw, not deserialized payloads
+      def raw_payloads
+        map(&:raw_payload)
+      end
+
+      # @return [Karafka::Messages::Message] first message
+      def first
+        @messages_array.first
+      end
+
+      # @return [Karafka::Messages::Message] last message
+      def last
+        @messages_array.last
+      end
+
+      # @return [Integer] number of messages in the batch
+      def size
+        @messages_array.size
+      end
+
+      # @return [Array<Karafka::Messages::Message>] pure array with messages
+      def to_a
+        @messages_array
+      end
+    end
+  end
+end

data/lib/karafka/{params → messages}/metadata.rb

@@ -1,18 +1,16 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Params
-    # Single message / params metadata details that can be accessed without the need for the
-    # payload deserialization
+  module Messages
+    # Single message metadata details that can be accessed without the need of deserialization.
     Metadata = Struct.new(
-      :create_time,
+      :timestamp,
       :headers,
-      :is_control_record,
       :key,
       :offset,
       :deserializer,
       :partition,
-      :receive_time,
+      :received_at,
       :topic,
       keyword_init: true
     )

data/lib/karafka/messages/seek.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # "Fake" message that we use as an abstraction layer when seeking back.
+    # This allows us to encapsulate a seek with a simple abstraction
+    Seek = Struct.new(:topic, :partition, :offset)
+  end
+end

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Patches to external components
+  module Patches
+    # Rdkafka related patches
+    module Rdkafka
+      # Rdkafka::Consumer patches
+      module Consumer
+        # A method that allows us to get the native kafka producer name
+        # @return [String] producer instance name
+        # @note We need this to make sure that we allocate proper dispatched events only to
+        #   callback listeners that should publish them
+        def name
+          ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Consumer.include ::Karafka::Patches::Rdkafka::Consumer