karafka 2.0.40 → 2.1.0

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for instrumentation related with Kubernetes
+      module Kubernetes
+        # Kubernetes HTTP listener that does not only reply when process is not fully hanging, but
+        # also allows to define max time of processing and looping.
+        #
+        # Processes like Karafka server can hang while still being reachable. For example, in case
+        # something would hang inside of the user code, Karafka could stop polling and no new
+        # data would be processed, but process itself would still be active. This listener allows
+        # for defining of a ttl that gets bumped on each poll loop and before and after processing
+        # of a given messages batch.
+        class LivenessListener
+          include ::Karafka::Core::Helpers::Time
+
+          # @param hostname [String, nil] hostname or nil to bind on all
+          # @param port [Integer] TCP port on which we want to run our HTTP status server
+          # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
+          #   It allows us to define max consumption time after which k8s should consider given
+          #   process as hanging
+          # @param polling_ttl [Integer] max time in ms for polling. If polling (any) does not
+          #   happen that often, process should be considered dead.
+          # @note The default TTL matches the default `max.poll.interval.ms`
+          def initialize(
+            hostname: nil,
+            port: 3000,
+            consuming_ttl: 5 * 60 * 1_000,
+            polling_ttl: 5 * 60 * 1_000
+          )
+            @server = TCPServer.new(*[hostname, port].compact)
+            @polling_ttl = polling_ttl
+            @consuming_ttl = consuming_ttl
+            @mutex = Mutex.new
+            @pollings = {}
+            @consumptions = {}
+
+            Thread.new do
+              loop do
+                break unless respond
+              end
+            end
+          end
+
+          # Tick on each fetch
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_fetch_loop(_event)
+            mark_polling_tick
+          end
+
+          # Tick on starting work
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consume(_event)
+            mark_consumption_tick
+          end
+
+          # Tick on finished work
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(_event)
+            clear_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoke(_event)
+            mark_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(_event)
+            clear_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutting_down(_event)
+            mark_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(_event)
+            clear_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(_event)
+            clear_consumption_tick
+            clear_polling_tick
+          end
+
+          # Stop the http server when we stop the process
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_stopped(_event)
+            @server.close
+          end
+
+          private
+
+          # Wraps the logic with a mutex
+          # @param block [Proc] code we want to run in mutex
+          def synchronize(&block)
+            @mutex.synchronize(&block)
+          end
+
+          # @return [Integer] object id of the current thread
+          def thread_id
+            Thread.current.object_id
+          end
+
+          # Update the polling tick time for current thread
+          def mark_polling_tick
+            synchronize do
+              @pollings[thread_id] = monotonic_now
+            end
+          end
+
+          # Clear current thread polling time tracker
+          def clear_polling_tick
+            synchronize do
+              @pollings.delete(thread_id)
+            end
+          end
+
+          # Update the processing tick time
+          def mark_consumption_tick
+            synchronize do
+              @consumptions[thread_id] = monotonic_now
+            end
+          end
+
+          # Clear current thread consumption time tracker
+          def clear_consumption_tick
+            synchronize do
+              @consumptions.delete(thread_id)
+            end
+          end
+
+          # Responds to a HTTP request with the process liveness status
+          def respond
+            client = @server.accept
+            client.gets
+            client.print "HTTP/1.1 #{status}\r\n"
+            client.close
+
+            true
+          rescue Errno::ECONNRESET, Errno::EPIPE, IOError
+            !@server.closed?
+          end
+
+          # Did we exceed any of the ttls
+          # @return [String] 204 string if ok, 500 otherwise
+          def status
+            time = monotonic_now
+
+            return '500' if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+            return '500' if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+
+            '204'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/consumer.rb

@@ -35,18 +35,9 @@ module Karafka
             # double-processing
             break if Karafka::App.stopping? && !topic.virtual_partitions?
 
-            # Break if we already know, that one of virtual partitions has failed and we will
-            # be restarting processing all together after all VPs are done. This will minimize
-            # number of jobs that will be re-processed
-            break if topic.virtual_partitions? && failing?
-
             consume_job(message)
 
-            # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
-            # this could create random markings.
-            # The exception here is the collapsed state where we can move one after another
-            next if topic.virtual_partitions? && !collapsed?
-
+            # We can always mark because of the virtual offset management that we have in VPs
             mark_as_consumed(message)
           end
         end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -39,7 +39,7 @@ module Karafka
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(job.serialize)
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           )
         end
@@ -54,7 +54,7 @@ module Karafka
 
             dispatches[d_method] << dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(job.serialize)
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           end
 

data/lib/karafka/pro/iterator.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Topic iterator allows you to iterate over topic/partition data and perform lookups for
+    # information that you need.
+    #
+    # It supports early stops on finding the requested data and allows for seeking till
+    # the end. It also allows for signaling, when a given message should be last out of certain
+    # partition, but we still want to continue iterating in other messages.
+    #
+    # It does **not** create a consumer group and does not have any offset management.
+    class Iterator
+      # Local partition reference for librdkafka
+      Partition = Struct.new(:partition, :offset)
+
+      private_constant :Partition
+
+      # A simple API allowing to iterate over topic/partition data, without having to subscribe
+      # and deal with rebalances. This API allows for multi-partition streaming and is optimized
+      # for data lookups. It allows for explicit stopping iteration over any partition during
+      # the iteration process, allowing for optimized lookups.
+      #
+      # @param topics [Array<String>, Hash] list of strings if we want to subscribe to multiple
+      #   topics and all of their partitions or a hash where keys are the topics and values are
+      #   hashes with partitions and their initial offsets.
+      # @param settings [Hash] extra settings for the consumer. Please keep in mind, that if
+      #   overwritten, you may want to include `auto.offset.reset` to match your case.
+      # @param yield_nil [Boolean] should we yield also `nil` values when poll returns nothing.
+      #   Useful in particular for long-living iterators.
+      #
+      # @note It is worth keeping in mind, that this API also needs to operate within
+      #   `max.poll.interval.ms` limitations on each iteration
+      #
+      # @note In case of a never-ending iterator, you need to set `enable.partition.eof` to `false`
+      #   so we don't stop polling data even when reaching the end (end on a given moment)
+      def initialize(
+        topics,
+        settings: { 'auto.offset.reset': 'beginning' },
+        yield_nil: false
+      )
+        @topics_with_partitions = expand_topics_with_partitions(topics)
+
+        @routing_topics = @topics_with_partitions.map do |name, _|
+          [name, ::Karafka::Routing::Router.find_or_initialize_by_name(name)]
+        end.to_h
+
+        @total_partitions = @topics_with_partitions.map(&:last).sum(&:count)
+
+        @stopped_partitions = 0
+
+        @settings = settings
+        @yield_nil = yield_nil
+      end
+
+      # Iterates over requested topic partitions and yields the results with the iterator itself
+      # Iterator instance is yielded because one can run `stop_partition` to stop iterating over
+      # part of data. It is useful for scenarios where we are looking for some information in all
+      # the partitions but once we found it, given partition data is no longer needed and would
+      # only eat up resources.
+      def each
+        Admin.with_consumer(@settings) do |consumer|
+          tpl = tpl_with_expanded_offsets(consumer)
+          consumer.assign(tpl)
+
+          # We need this for self-referenced APIs like pausing
+          @current_consumer = consumer
+
+          # Stream data until we reach the end of all the partitions or until the end user
+          # indicates that they are done
+          until done?
+            message = poll(200)
+
+            # Skip nils if not explicitly required
+            next if message.nil? && !@yield_nil
+
+            if message
+              @current_message = build_message(message)
+
+              yield(@current_message, self)
+            else
+              yield(nil, self)
+            end
+          end
+
+          @current_message = nil
+          @current_consumer = nil
+        end
+
+        # Reset so we can use the same iterator again if needed
+        @stopped_partitions = 0
+      end
+
+      # Stops the partition we're currently yielded into
+      def stop_current_partition
+        stop_partition(
+          @current_message.topic,
+          @current_message.partition
+        )
+      end
+
+      # Stops processing of a given partition
+      # We expect the partition to be provided because of a scenario, where there is a
+      # multi-partition iteration and we want to stop a different partition that the one that
+      # is currently yielded.
+      #
+      # We pause it forever and no longer work with it.
+      #
+      # @param name [String] topic name of which partition we want to stop
+      # @param partition [Integer] partition we want to stop processing
+      def stop_partition(name, partition)
+        @stopped_partitions += 1
+
+        @current_consumer.pause(
+          Rdkafka::Consumer::TopicPartitionList.new(
+            name => [Partition.new(partition, 0)]
+          )
+        )
+      end
+
+      private
+
+      # Expands topics to which we want to subscribe with partitions information in case this
+      # info is not provided. For our convenience we want to support 5 formats of defining
+      # the subscribed topics:
+      #
+      # - 'topic1' - just a string with one topic name
+      # - ['topic1', 'topic2'] - just the names
+      # - { 'topic1' => -100 } - names with negative lookup offset
+      # - { 'topic1' => { 0 => 5 } } - names with exact partitions offsets
+      # - { 'topic1' => { 0 => -5 }, 'topic2' => { 1 => 5 } } - with per partition negative offsets
+      #
+      # @param topics [Array, Hash] topics definitions
+      # @return [Hash] hash with topics containing partitions definitions
+      def expand_topics_with_partitions(topics)
+        # Simplification for the single topic case
+        topics = [topics] if topics.is_a?(String)
+        # If we've got just array with topics, we need to convert that into a representation
+        # that we can expand with offsets
+        topics = topics.map { |name| [name, false] }.to_h if topics.is_a?(Array)
+
+        expanded = Hash.new { |h, k| h[k] = {} }
+
+        topics.map do |topic, details|
+          if details.is_a?(Hash)
+            details.each do |partition, offset|
+              expanded[topic][partition] = offset
+            end
+          else
+            partition_count(topic.to_s).times do |partition|
+              # If no offsets are provided, we just start from zero
+              expanded[topic][partition] = details || 0
+            end
+          end
+        end
+
+        expanded
+      end
+
+      # @param timeout [Integer] timeout in ms
+      # @return [Rdkafka::Consumer::Message, nil] message or nil if nothing to do
+      def poll(timeout)
+        @current_consumer.poll(timeout)
+      rescue Rdkafka::RdkafkaError => e
+        # End of partition
+        if e.code == :partition_eof
+          @stopped_partitions += 1
+
+          retry
+        end
+
+        raise e
+      end
+
+      # Converts raw rdkafka message into Karafka message
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [::Karafka::Messages::Message]
+      def build_message(message)
+        Messages::Builders::Message.call(
+          message,
+          @routing_topics.fetch(message.topic),
+          Time.now
+        )
+      end
+
+      # Do we have all the data we wanted or did every topic partition has reached eof.
+      # @return [Boolean]
+      def done?
+        @stopped_partitions >= @total_partitions
+      end
+
+      # Builds the tpl representing all the subscriptions we want to run
+      #
+      # Additionally for negative offsets, does the watermark calculation where to start
+      #
+      # @param consumer [Rdkafka::Consumer] consumer we need in case of negative offsets as
+      #   negative are going to be used to do "give me last X". We use the already initialized
+      #   consumer instance, not to start another one again.
+      # @return [Rdkafka::Consumer::TopicPartitionList]
+      def tpl_with_expanded_offsets(consumer)
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+        @topics_with_partitions.each do |name, partitions|
+          partitions_with_offsets = {}
+
+          # When no offsets defined, we just start from zero
+          if partitions.is_a?(Array) || partitions.is_a?(Range)
+            partitions_with_offsets = partitions.map { |partition| [partition, 0] }.to_h
+          else
+            # When offsets defined, we can either use them if positive or expand and move back
+            # in case of negative (-1000 means last 1000 messages, etc)
+            partitions.each do |partition, offset|
+              if offset.negative?
+                _, high_watermark_offset = consumer.query_watermark_offsets(name, partition)
+                # We add because this offset is negative
+                partitions_with_offsets[partition] = high_watermark_offset + offset
+              else
+                partitions_with_offsets[partition] = offset
+              end
+            end
+          end
+
+          tpl.add_topic_and_partitions_with_offsets(name, partitions_with_offsets)
+        end
+
+        tpl
+      end
+
+      # @param name [String] topic name
+      # @return [Integer] number of partitions of the topic we want to iterate over
+      def partition_count(name)
+        Admin
+          .cluster_info
+          .topics
+          .find { |topic| topic.fetch(:topic_name) == name }
+          .fetch(:partitions)
+          .count
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/coordinator.rb

@@ -17,7 +17,7 @@ module Karafka
       # Pro coordinator that provides extra orchestration methods useful for parallel processing
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
-        attr_reader :filter
+        attr_reader :filter, :virtual_offset_manager
 
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
@@ -27,6 +27,20 @@ module Karafka
           @flow_lock = Mutex.new
           @collapser = Collapser.new
           @filter = FiltersApplier.new(self)
+
+          return unless topic.virtual_partitions?
+
+          @virtual_offset_manager = VirtualOffsetManager.new(
+            topic.name,
+            partition
+          )
+
+          # We register our own "internal" filter to support filtering of messages that were marked
+          # as consumed virtually
+          @filter.filters << Filters::VirtualLimiter.new(
+            @virtual_offset_manager,
+            @collapser
+          )
         end
 
         # Starts the coordination process
@@ -40,6 +54,11 @@ module Karafka
           @filter.apply!(messages)
 
           @executed.clear
+
+          # We keep the old processed offsets until the collapsing is done and regular processing
+          # with virtualization is restored
+          @virtual_offset_manager.clear if topic.virtual_partitions? && !@collapser.collapsed?
+
           @last_message = messages.last
         end
 

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Removes messages that are already marked as consumed in the virtual offset manager
+        # This should operate only when using virtual partitions.
+        #
+        # This cleaner prevents us from duplicated processing of messages that were virtually
+        # marked as consumed even if we could not mark them as consumed in Kafka. This allows us
+        # to limit reprocessing when errors occur drastically when operating with virtual
+        # partitions
+        #
+        # @note It should be registered only when VPs are used
+        class VirtualLimiter < Base
+          # @param manager [Processing::VirtualOffsetManager]
+          # @param collapser [Processing::Collapser]
+          def initialize(manager, collapser)
+            @manager = manager
+            @collapser = collapser
+
+            super()
+          end
+
+          # Remove messages that we already marked as virtually consumed. Does nothing if not in
+          # the collapsed mode.
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            return unless @collapser.collapsed?
+
+            marked = @manager.marked
+
+            messages.delete_if { |message| marked.include?(message.offset) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/filters_applier.rb

@@ -21,6 +21,10 @@ module Karafka
       # This means that this is the API we expose as a single filter, allowing us to control
       # the filtering via many filters easily.
       class FiltersApplier
+        # @return [Array] registered filters array. Useful if we want to inject internal context
+        #   aware filters.
+        attr_reader :filters
+
         # @param coordinator [Pro::Coordinator] pro coordinator
         def initialize(coordinator)
           # Builds filters out of their factories

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -22,7 +22,7 @@ module Karafka
           # - Mom
           # - VP
           module DlqFtrMomVp
-            include Strategies::Vp::Default
+            include Strategies::Aj::DlqMomVp
             include Strategies::Aj::DlqFtrMom
 
             # Features for this strategy

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -24,7 +24,9 @@ module Karafka
           # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
           # offset management works in this case.
           module DlqLrjMom
-            include Strategies::Aj::DlqLrjMomVp
+            include Strategies::Default
+            include Strategies::Dlq::Default
+            include Strategies::Aj::LrjMom
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -20,9 +20,9 @@ module Karafka
           # Manual offset management enabled
           # Virtual Partitions enabled
           module DlqMomVp
-            include Strategies::Dlq::Default
-            include Strategies::Vp::Default
             include Strategies::Default
+            include Strategies::Dlq::Vp
+            include Strategies::Vp::Default
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -34,6 +34,8 @@ module Karafka
 
             # No actions needed for the standard flow here
             def handle_before_enqueue
+              super
+
               coordinator.on_enqueued do
                 pause(coordinator.seek_offset, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
               end

data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb

@@ -20,8 +20,8 @@ module Karafka
           # Manual offset management enabled
           # Virtual Partitions enabled
           module MomVp
-            include Strategies::Vp::Default
             include Strategies::Default
+            include Strategies::Vp::Default
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -20,7 +20,7 @@ module Karafka
           # - Ftr
           module Ftr
             include Strategies::Ftr::Default
-            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Default
 
             # Features for this strategy
             FEATURES = %i[

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -55,14 +55,11 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, marked = find_skippable_message
+                  skippable_message, _marked = find_skippable_message
                   dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                  if marked
-                    pause(coordinator.seek_offset, nil, false)
-                  else
-                    pause(skippable_message.offset + 1, nil, false)
-                  end
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
                 end
               end
             end