karafka 2.3.4 → 2.4.0.beta1

data/lib/karafka/instrumentation/logger_listener.rb

@@ -16,10 +16,20 @@ module Karafka
 
       private_constant :USED_LOG_LEVELS
 
+      # @param log_polling [Boolean] should we log the fact that messages are being polled. This is
+      #   usually noisy and not useful in production but can be useful in dev. While users can
+      #   do this themselves this has been requested and asked for often, thus similar to how
+      #   extensive logging can be disabled in WaterDrop, we do it here as well.
+      def initialize(log_polling: true)
+        @log_polling = log_polling
+      end
+
       # Logs each messages fetching attempt
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop(event)
+        return unless log_polling?
+
         listener = event[:caller]
         debug "[#{listener.id}] Polling messages..."
       end
@@ -28,6 +38,8 @@ module Karafka
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop_received(event)
+        return unless log_polling?
+
         listener = event[:caller]
         time = event[:time]
         messages_count = event[:messages_buffer].size
@@ -129,7 +141,7 @@ module Karafka
         Thread.list.each do |thread|
           tid = (thread.object_id ^ ::Process.pid).to_s(36)
 
-          warn "Thread TID-#{tid} #{thread['label']}"
+          warn "Thread TID-#{tid} #{thread.name}"
 
           if thread.backtrace
             warn thread.backtrace.join("\n")
@@ -315,9 +327,10 @@ module Karafka
         when 'connection.client.unsubscribe.error'
           error "Client unsubscribe error occurred: #{error}"
           error details
+        # This handles any custom errors coming from places like Web-UI, etc
         else
-          # This should never happen. Please contact the maintainers
-          raise Errors::UnsupportedCaseError, event
+          error "#{type} error occurred: #{error}"
+          error details
         end
       end
 
@@ -326,6 +339,13 @@ module Karafka
           Karafka.logger.send(log_level, *args)
         end
       end
+
+      private
+
+      # @return [Boolean] should we log polling
+      def log_polling?
+        @log_polling
+      end
     end
   end
 end

data/lib/karafka/instrumentation/notifications.rb

@@ -40,6 +40,14 @@ module Karafka
         connection.listener.fetch_loop.received
         connection.listener.after_fetch_loop
 
+        connection.listener.pending
+        connection.listener.starting
+        connection.listener.running
+        connection.listener.quieting
+        connection.listener.quiet
+        connection.listener.stopping
+        connection.listener.stopped
+
         consumer.before_schedule_consume
         consumer.consume
         consumer.consumed
@@ -66,6 +74,8 @@ module Karafka
         filtering.throttled
         filtering.seek
 
+        oauthbearer.token_refresh
+
         process.notice_signal
 
         rebalance.partitions_assign

data/lib/karafka/instrumentation/vendors/appsignal/client.rb

@@ -11,6 +11,13 @@ module Karafka
         #
         # @note This client is abstract, it has no notion of Karafka whatsoever
         class Client
+          # @param namespace_name [String, nil] Name of the AppSignal namespace we want to use or
+          #   nil if it is to remain default.
+          #   Defaults to `Appsignal::Transaction::BACKGROUND_JOB` in the execution flow.
+          def initialize(namespace_name: nil)
+            @namespace_name = namespace_name
+          end
+
           # Starts an appsignal transaction with a given action name
           #
           # @param action_name [String] action name. For processing this should be equal to
@@ -18,7 +25,7 @@ module Karafka
           def start_transaction(action_name)
             transaction = ::Appsignal::Transaction.create(
               SecureRandom.uuid,
-              ::Appsignal::Transaction::BACKGROUND_JOB,
+              namespace_name,
               ::Appsignal::Transaction::GenericRequest.new({})
             )
 
@@ -83,7 +90,7 @@ module Karafka
               transaction.set_error(error)
             else
               ::Appsignal.send_error(error) do |transaction|
-                transaction.set_namespace(::Appsignal::Transaction::BACKGROUND_JOB)
+                transaction.set_namespace(namespace_name)
               end
             end
           end
@@ -115,6 +122,13 @@ module Karafka
               .transform_values(&:to_s)
               .transform_keys!(&:to_s)
           end
+
+          # @return [String] transaction namespace. We lazy evaluate it and resolve if needed to
+          #   the default `BACKGROUND_JOB` during the execution, to ensure we can initialize the
+          #   instrumentation even before appsignal gem is loaded.
+          def namespace_name
+            @namespace_name ||= ::Appsignal::Transaction::BACKGROUND_JOB
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -92,6 +92,26 @@ module Karafka
             clear_polling_tick
           end
 
+          # Deregister the polling tracker for given listener
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_stopping(_event)
+            # We are interested in disabling tracking for given listener only if it was requested
+            # when karafka was running. If we would always clear, it would not catch the shutdown
+            # polling requirements. The "running" listener shutdown operations happen only when
+            # the manager requests it for downscaling.
+            return if Karafka::App.done?
+
+            clear_polling_tick
+          end
+
+          # Deregister the polling tracker for given listener
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_stopped(_event)
+            return if Karafka::App.done?
+
+            clear_polling_tick
+          end
+
           private
 
           # Wraps the logic with a mutex

data/lib/karafka/messages/batch_metadata.rb

@@ -10,7 +10,7 @@ module Karafka
       :size,
       :first_offset,
       :last_offset,
-      :deserializer,
+      :deserializers,
       :partition,
       :topic,
       :created_at,

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -21,7 +21,7 @@ module Karafka
               size: messages.count,
               first_offset: messages.first&.offset || -1001,
               last_offset: messages.last&.offset || -1001,
-              deserializer: topic.deserializer,
+              deserializers: topic.deserializers,
               partition: partition,
               topic: topic.name,
               # We go with the assumption that the creation of the whole batch is the last message

data/lib/karafka/messages/builders/message.rb

@@ -14,14 +14,14 @@ module Karafka
           def call(kafka_message, topic, received_at)
             metadata = Karafka::Messages::Metadata.new(
               timestamp: kafka_message.timestamp,
-              headers: kafka_message.headers,
-              key: kafka_message.key,
               offset: kafka_message.offset,
-              deserializer: topic.deserializer,
+              deserializers: topic.deserializers,
               partition: kafka_message.partition,
               topic: topic.name,
-              received_at: received_at
-            ).freeze
+              received_at: received_at,
+              raw_headers: kafka_message.headers,
+              raw_key: kafka_message.key
+            )
 
             # Get the raw payload
             payload = kafka_message.payload
@@ -31,7 +31,11 @@ module Karafka
             kafka_message.instance_variable_set('@payload', nil)
 
             # Karafka messages cannot be frozen because of the lazy deserialization feature
-            Karafka::Messages::Message.new(payload, metadata)
+            message = Karafka::Messages::Message.new(payload, metadata)
+            # Assign message to metadata so we can reverse its relationship if needed
+            metadata[:message] = message
+
+            message
           end
         end
       end

data/lib/karafka/messages/message.rb

@@ -23,7 +23,8 @@ module Karafka
       # prior to the final deserialization
       attr_accessor :raw_payload
 
-      def_delegators :metadata, *Metadata.members
+      # We remove message as we do not want to do self-reference via `message.message`
+      def_delegators :metadata, *((Metadata.members + %i[key headers]) - %i[message])
 
       # @param raw_payload [Object] incoming payload before deserialization
       # @param metadata [Karafka::Messages::Metadata] message metadata object

data/lib/karafka/messages/metadata.rb

@@ -4,15 +4,31 @@ module Karafka
   module Messages
     # Single message metadata details that can be accessed without the need of deserialization.
     Metadata = Struct.new(
+      :message,
       :timestamp,
-      :headers,
-      :key,
       :offset,
-      :deserializer,
+      :deserializers,
       :partition,
       :received_at,
       :topic,
+      :raw_headers,
+      :raw_key,
       keyword_init: true
-    )
+    ) do
+      # @return [Object] deserialized key. By default in the raw string format.
+      def key
+        return @key if @key
+
+        @key = deserializers.key.call(self)
+      end
+
+      # @return [Object] deserialized headers. By default its a hash with keys and payload being
+      #   strings
+      def headers
+        return @headers if @headers
+
+        @headers = deserializers.headers.call(self)
+      end
+    end
   end
 end

data/lib/karafka/messages/parser.rb

@@ -7,7 +7,7 @@ module Karafka
       # @param message [::Karafka::Messages::Message]
       # @return [Object] deserialized payload
       def call(message)
-        message.metadata.deserializer.call(message)
+        message.metadata.deserializers.payload.call(message)
       end
     end
   end

data/lib/karafka/pro/base_consumer.rb

@@ -20,29 +20,6 @@ module Karafka
     #
     # Methods here are suppose to be always available or are expected to be redefined
     module BaseConsumer
-      # Runs the on-schedule tick periodic operations
-      # This method is an alias but is part of the naming convention used for other flows, this
-      # is why we do not reference the `handle_before_schedule_tick` directly
-      def on_before_schedule_tick
-        handle_before_schedule_tick
-      end
-
-      # Used by the executor to trigger consumer tick
-      # @private
-      def on_tick
-        handle_tick
-      rescue StandardError => e
-        Karafka.monitor.instrument(
-          'error.occurred',
-          error: e,
-          caller: self,
-          type: 'consumer.tick.error'
-        )
-      end
-
-      # By default we do nothing when ticking
-      def tick; end
-
       # @return [Karafka::Pro::Processing::Coordinators::ErrorsTracker] tracker for errors that
       #   occurred during processing until another successful processing
       #
@@ -58,6 +35,18 @@ module Karafka
       def errors_tracker
         coordinator.errors_tracker
       end
+
+      # @return [Karafka::Pro::Processing::SubscriptionGroupsCoordinator] Coordinator allowing to
+      #   pause and resume polling of the given subscription group jobs queue for postponing
+      #   further work.
+      #
+      # @note Since this stops polling, it can cause reaching `max.poll.interval.ms` limitations.
+      #
+      # @note This is a low-level API used for cross-topic coordination and some advanced features.
+      #   Use it at own risk.
+      def subscription_groups_coordinator
+        Processing::SubscriptionGroupsCoordinator.instance
+      end
     end
   end
 end

data/lib/karafka/pro/encryption/cipher.rb

@@ -16,6 +16,10 @@ module Karafka
     module Encryption
       # Cipher for encrypting and decrypting data
       class Cipher
+        include Helpers::ConfigImporter.new(
+          encryption: %i[encryption]
+        )
+
         def initialize
           @private_pems = {}
         end
@@ -39,7 +43,7 @@ module Karafka
 
         # @return [::OpenSSL::PKey::RSA] rsa public key
         def public_pem
-          @public_pem ||= ::OpenSSL::PKey::RSA.new(::Karafka::App.config.encryption.public_key)
+          @public_pem ||= ::OpenSSL::PKey::RSA.new(encryption.public_key)
         end
 
         # @param version [String] version for which we want to get the rsa key
@@ -47,8 +51,8 @@ module Karafka
         def private_pem(version)
           return @private_pems[version] if @private_pems.key?(version)
 
-          key_string = ::Karafka::App.config.encryption.private_keys[version]
-          key_string || raise(Errors::PrivateKeyNotFound, version)
+          key_string = encryption.private_keys[version]
+          key_string || raise(Errors::PrivateKeyNotFoundError, version)
 
           @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
         end

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -30,6 +30,7 @@ module Karafka
             required(:active) { |val| [true, false].include?(val) }
             required(:version) { |val| val.is_a?(String) && !val.empty? }
             required(:public_key) { |val| val.is_a?(String) }
+            required(:fingerprinter) { |val| val == false || val.respond_to?(:hexdigest) }
 
             required(:private_keys) do |val|
               val.is_a?(Hash) &&

data/lib/karafka/pro/encryption/errors.rb

@@ -20,7 +20,10 @@ module Karafka
         BaseError = Class.new(::Karafka::Errors::BaseError)
 
         # Raised when we have encountered encryption key with version we do not have
-        PrivateKeyNotFound = Class.new(BaseError)
+        PrivateKeyNotFoundError = Class.new(BaseError)
+
+        # Raised when fingerprinting was enabled and payload after encryption did not match it
+        FingerprintVerificationError = Class.new(BaseError)
       end
     end
   end

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -18,26 +18,28 @@ module Karafka
       module Messages
         # Middleware for WaterDrop. It automatically encrypts messages payload.
         # It is injected only if encryption is enabled.
+        # It also fingerprints the payload for verification if fingerprinting was enabled
         class Middleware
+          include Helpers::ConfigImporter.new(
+            cipher: %i[encryption cipher],
+            version: %i[encryption version],
+            fingerprinter: %i[encryption fingerprinter]
+          )
+
           # @param message [Hash] WaterDrop message hash
           # @return [Hash] hash with encrypted payload and encryption version indicator
           def call(message)
+            payload = message[:payload]
+
             message[:headers] ||= {}
             message[:headers]['encryption'] = version
-            message[:payload] = cipher.encrypt(message[:payload])
-            message
-          end
+            message[:payload] = cipher.encrypt(payload)
 
-          private
+            return message unless fingerprinter
 
-          # @return [::Karafka::Pro::Encryption::Cipher]
-          def cipher
-            @cipher ||= ::Karafka::App.config.encryption.cipher
-          end
+            message[:headers]['encryption_fingerprint'] = fingerprinter.hexdigest(payload)
 
-          # @return [String] encryption version
-          def version
-            @version ||= ::Karafka::App.config.encryption.version
+            message
           end
         end
       end

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -20,34 +20,36 @@ module Karafka
         #   unencrypted payloads. That is why we always rely on message headers for encryption
         #   indication.
         class Parser < ::Karafka::Messages::Parser
+          include Helpers::ConfigImporter.new(
+            cipher: %i[encryption cipher],
+            active: %i[encryption active],
+            fingerprinter: %i[encryption fingerprinter]
+          )
+
           # @param message [::Karafka::Messages::Message]
           # @return [Object] deserialized payload
           def call(message)
-            if active? && message.headers.key?('encryption')
-              # Decrypt raw payload so it can be handled by the default parser logic
-              message.raw_payload = cipher.decrypt(
-                message.headers['encryption'],
-                message.raw_payload
-              )
-            end
-
-            super(message)
-          end
+            headers = message.headers
+            encryption = headers['encryption']
+            fingerprint = headers['encryption_fingerprint']
 
-          private
+            return super(message) unless active && encryption
 
-          # @return [::Karafka::Pro::Encryption::Cipher]
-          def cipher
-            @cipher ||= ::Karafka::App.config.encryption.cipher
-          end
+            # Decrypt raw payload so it can be handled by the default parser logic
+            decrypted_payload = cipher.decrypt(
+              encryption,
+              message.raw_payload
+            )
+
+            message.raw_payload = decrypted_payload
+
+            return super(message) unless fingerprint && fingerprinter
 
-          # @return [Boolean] is encryption active
-          def active?
-            return @active unless @active.nil?
+            message_fingerprint = fingerprinter.hexdigest(decrypted_payload)
 
-            @active = ::Karafka::App.config.encryption.active
+            return super(message) if message_fingerprint == fingerprint
 
-            @active
+            raise(Errors::FingerprintVerificationError, message.to_s)
           end
         end
       end

data/lib/karafka/pro/encryption/setup/config.rb

@@ -40,6 +40,11 @@ module Karafka
           # Cipher used to encrypt and decrypt data
           setting(:cipher, default: Encryption::Cipher.new)
 
+          # When set to any digest that responds to `#hexdigest` will compute checksum of the
+          # message payload for post-description integrity verification. It will include a
+          # fingerprint in headers
+          setting(:fingerprinter, default: false)
+
           configure
         end
       end

data/lib/karafka/pro/iterator/expander.rb

@@ -29,7 +29,8 @@ module Karafka
       # - { 'topic1' => 100 } - means we run all partitions from the offset 100
       # - { 'topic1' => Time.now - 60 } - we run all partitions from the message from 60s ago
       # - { 'topic1' => { 1 => Time.now - 60 } } - partition1 from message 60s ago
-      #
+      # - { 'topic1' => { 1 => true } } - will pick first offset not consumed on this CG for p 1
+      # - { 'topic1' => true } - will pick first offset not consumed on this CG for all p
       class Expander
         # Expands topics to which we want to subscribe with partitions information in case this
         # info is not provided.

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -36,6 +36,7 @@ module Karafka
           resolve_partitions_with_exact_offsets
           resolve_partitions_with_negative_offsets
           resolve_partitions_with_time_offsets
+          resolve_partitions_with_cg_expectations
 
           # Final tpl with all the data
           tpl = Rdkafka::Consumer::TopicPartitionList.new
@@ -149,6 +150,43 @@ module Karafka
             end
           end
         end
+
+        # Fetches last used offsets for those partitions for which we want to consume from last
+        # moment where given consumer group has finished
+        # This is indicated by given partition value being set to `true`.
+        def resolve_partitions_with_cg_expectations
+          tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+          # First iterate over all topics that we want to expand
+          @expanded_topics.each do |name, partitions|
+            partitions_base = {}
+
+            partitions.each do |partition, offset|
+              # Pick only partitions where offset is set to true to indicate that we are interested
+              # in committed offset resolution
+              next unless offset == true
+
+              # This can be set to nil because we do not use this offset value when querying
+              partitions_base[partition] = nil
+            end
+
+            # If there is nothing to work with, just skip
+            next if partitions_base.empty?
+
+            tpl.add_topic_and_partitions_with_offsets(name, partitions_base)
+          end
+
+          # If nothing to resolve, do not resolve
+          return if tpl.empty?
+
+          # Fetch all committed offsets for all the topics partitions of our interest and use
+          # those offsets for the mapped topics data
+          @consumer.committed(tpl).to_h.each do |name, partitions|
+            partitions.each do |partition|
+              @mapped_topics[name][partition.partition] = partition.offset
+            end
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/iterator.rb

@@ -20,7 +20,9 @@ module Karafka
     # the end. It also allows for signaling, when a given message should be last out of certain
     # partition, but we still want to continue iterating in other messages.
     #
-    # It does **not** create a consumer group and does not have any offset management.
+    # It does **not** create a consumer group and does not have any offset management until first
+    # consumer offset marking happens. So can be use for quick seeks as well as iterative,
+    # repetitive data fetching from rake, etc.
     class Iterator
       # A simple API allowing to iterate over topic/partition data, without having to subscribe
       # and deal with rebalances. This API allows for multi-partition streaming and is optimized
@@ -92,6 +94,7 @@ module Karafka
             end
           end
 
+          @current_consumer.commit_offsets(async: false) if @stored_offsets
           @current_message = nil
           @current_consumer = nil
         end
@@ -127,6 +130,29 @@ module Karafka
         )
       end
 
+      # Stops all the iterating
+      # @note `break` can also be used but in such cases commits stored async will not be flushed
+      #   to Kafka. This is why `#stop` is the recommended method.
+      def stop
+        @stopped = true
+      end
+
+      # Marks given message as consumed.
+      #
+      # @param message [Karafka::Messages::Message] message that we want to mark as processed
+      def mark_as_consumed(message)
+        @current_consumer.store_offset(message, nil)
+        @stored_offsets = true
+      end
+
+      # Marks given message as consumed and commits offsets
+      #
+      # @param message [Karafka::Messages::Message] message that we want to mark as processed
+      def mark_as_consumed!(message)
+        mark_as_consumed(message)
+        @current_consumer.commit_offsets(async: false)
+      end
+
       private
 
       # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
@@ -158,7 +184,7 @@ module Karafka
       # Do we have all the data we wanted or did every topic partition has reached eof.
       # @return [Boolean]
       def done?
-        @stopped_partitions >= @total_partitions
+        (@stopped_partitions >= @total_partitions) || @stopped
       end
     end
   end

data/lib/karafka/pro/loader.rb

@@ -63,6 +63,9 @@ module Karafka
         # @param config [Karafka::Core::Configurable::Node]
         def post_setup_all(config)
           features.each { |feature| feature.post_setup(config) }
+
+          # We initialize it here so we don't initialize it during multi-threading work
+          Processing::SubscriptionGroupsCoordinator.instance
         end
 
         private

data/lib/karafka/pro/processing/coordinator.rb

@@ -99,8 +99,9 @@ module Karafka
         end
 
         # @return [Boolean] is the coordinated work finished or not
+        # @note Used only in the consume operation context
         def finished?
-          @running_jobs.zero?
+          @running_jobs[:consume].zero?
         end
 
         # Runs synchronized code once for a collective of virtual partitions prior to work being
@@ -122,7 +123,7 @@ module Karafka
           end
         end
 
-        # Runs once when all the work that is suppose to be coordinated is finished
+        # Runs given code once when all the work that is suppose to be coordinated is finished
         # It runs once per all the coordinated jobs and should be used to run any type of post
         # jobs coordination processing execution
         def on_finished
@@ -143,6 +144,18 @@ module Karafka
           end
         end
 
+        # @param interval [Integer] milliseconds of activity
+        # @return [Boolean] was this partition in activity within last `interval` milliseconds
+        # @note Will return true also if currently active
+        def active_within?(interval)
+          # its always active if there's any job related to this coordinator that is still
+          # enqueued or running
+          return true if @running_jobs.values.any?(:positive?)
+
+          # Otherwise we check last time any job of this coordinator was active
+          @changed_at + interval > monotonic_now
+        end
+
         private
 
         # Checks if given action is executable once. If it is and true is returned, this method

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -23,7 +23,9 @@ module Karafka
         def find(topic)
           # Start with the non-pro expansions
           expansions = super
+          expansions << Pro::Processing::Piping::Consumer
           expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
+          expansions << Pro::Processing::PeriodicJob::Consumer if topic.periodic_job?
           expansions
         end
       end