karafka 2.3.1 → 2.3.2

data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for instrumentation related with Kubernetes
+      module Kubernetes
+        # Base Kubernetes Listener providing basic HTTP server capabilities to respond with health
+        class BaseListener
+          include ::Karafka::Core::Helpers::Time
+
+          # All good with Karafka
+          OK_CODE = '204 No Content'
+
+          # Some timeouts, fail
+          FAIL_CODE = '500 Internal Server Error'
+
+          private_constant :OK_CODE, :FAIL_CODE
+
+          # @param hostname [String, nil] hostname or nil to bind on all
+          # @param port [Integer] TCP port on which we want to run our HTTP status server
+          def initialize(
+            hostname: nil,
+            port: 3000
+          )
+            @hostname = hostname
+            @port = port
+          end
+
+          private
+
+          # @return [Boolean] true if all good, false if we should tell k8s to kill this process
+          def healthy?
+            raise NotImplementedError, 'Implement in a subclass'
+          end
+
+          # Responds to a HTTP request with the process liveness status
+          def respond
+            client = @server.accept
+            client.gets
+            client.print "HTTP/1.1 #{healthy? ? OK_CODE : FAIL_CODE}\r\n"
+            client.print "Content-Type: text/plain\r\n"
+            client.print "\r\n"
+            client.close
+
+            true
+          rescue Errno::ECONNRESET, Errno::EPIPE, IOError
+            !@server.closed?
+          end
+
+          # Starts background thread with micro-http monitoring
+          def start
+            @server = TCPServer.new(*[@hostname, @port].compact)
+
+            Thread.new do
+              loop do
+                break unless respond
+              end
+            end
+          end
+
+          # Stops the server
+          def stop
+            @server.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'socket'
+require 'karafka/instrumentation/vendors/kubernetes/base_listener'
 
 module Karafka
   module Instrumentation
@@ -23,17 +23,9 @@ module Karafka
         #
         # @note In case of usage within an embedding with Puma, you need to select different port
         #   then the one used by Puma itself.
-        class LivenessListener
-          include ::Karafka::Core::Helpers::Time
-
-          # All good with Karafka
-          OK_CODE = '204 No Content'
-
-          # Some timeouts, fail
-          FAIL_CODE = '500 Internal Server Error'
-
-          private_constant :OK_CODE, :FAIL_CODE
-
+        #
+        # @note Please use `Kubernetes::SwarmLivenessListener` when operating in the swarm mode
+        class LivenessListener < BaseListener
           # @param hostname [String, nil] hostname or nil to bind on all
           # @param port [Integer] TCP port on which we want to run our HTTP status server
           # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
@@ -48,30 +40,23 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
-            @hostname = hostname
-            @port = port
             @polling_ttl = polling_ttl
             @consuming_ttl = consuming_ttl
             @mutex = Mutex.new
             @pollings = {}
             @consumptions = {}
+            super(hostname: hostname, port: port)
           end
 
           # @param _event [Karafka::Core::Monitoring::Event]
           def on_app_running(_event)
-            @server = TCPServer.new(*[@hostname, @port].compact)
-
-            Thread.new do
-              loop do
-                break unless respond
-              end
-            end
+            start
           end
 
           # Stop the http server when we stop the process
           # @param _event [Karafka::Core::Monitoring::Event]
           def on_app_stopped(_event)
-            @server.close
+            stop
           end
 
           # Tick on each fetch
@@ -148,29 +133,15 @@ module Karafka
             end
           end
 
-          # Responds to a HTTP request with the process liveness status
-          def respond
-            client = @server.accept
-            client.gets
-            client.print "HTTP/1.1 #{status}\r\n"
-            client.print "Content-Type: text/plain\r\n"
-            client.print "\r\n"
-            client.close
-
-            true
-          rescue Errno::ECONNRESET, Errno::EPIPE, IOError
-            !@server.closed?
-          end
-
           # Did we exceed any of the ttls
           # @return [String] 204 string if ok, 500 otherwise
-          def status
+          def healthy?
             time = monotonic_now
 
-            return FAIL_CODE if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return FAIL_CODE if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
 
-            OK_CODE
+            true
           end
         end
       end

data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'karafka/instrumentation/vendors/kubernetes/base_listener'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      module Kubernetes
+        # Kubernetes HTTP listener designed to operate with Karafka running in the swarm mode
+        # In the Swarm mode we supervise only the supervisor as other nodes are suppose to be
+        # managed by the swarm supervisor
+        class SwarmLivenessListener < BaseListener
+          # @param hostname [String, nil] hostname or nil to bind on all
+          # @param port [Integer] TCP port on which we want to run our HTTP status server
+          # @param controlling_ttl [Integer] time in ms after which we consider the supervising
+          #   thread dead because it is not controlling nodes. When configuring this, please take
+          #   into consideration, that during shutdown of the swarm, there is no controlling
+          #   happening.
+          def initialize(
+            hostname: nil,
+            port: 3000,
+            controlling_ttl: 60 * 1_000
+          )
+            @hostname = hostname
+            @port = port
+            @controlling_ttl = controlling_ttl
+            @controlling = monotonic_now
+            super(port: port, hostname: hostname)
+          end
+
+          # Starts reporting in the supervisor only when it runs
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_supervising(_event)
+            start
+          end
+
+          # Tick on each control
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_swarm_manager_control(_event)
+            @controlling = monotonic_now
+          end
+
+          private
+
+          # Did we exceed any of the ttls
+          # @return [String] 204 string if ok, 500 otherwise
+          def healthy?
+            (monotonic_now - @controlling) < @controlling_ttl
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -26,7 +26,7 @@ module Karafka
         end
 
         optional(:partitioner) { |val| val.respond_to?(:call) }
-        optional(:partition_key_type) { |val| %i[key partition_key].include?(val) }
+        optional(:partition_key_type) { |val| %i[key partition_key partition].include?(val) }
         optional(:dispatch_method) do |val|
           %i[
             produce_async

data/lib/karafka/pro/base_consumer.rb

@@ -42,6 +42,22 @@ module Karafka
 
       # By default we do nothing when ticking
       def tick; end
+
+      # @return [Karafka::Pro::Processing::Coordinators::ErrorsTracker] tracker for errors that
+      #   occurred during processing until another successful processing
+      #
+      # @note This will always contain **only** details of errors that occurred during `#consume`
+      #   because only those are retryable.
+      #
+      # @note This may contain more than one error because:
+      #   - this can collect various errors that might have happened during virtual partitions
+      #     execution
+      #   - errors can pile up during retries and until a clean run, they will be collected with
+      #     a limit of last 100. We do not store more because a consumer with an endless error loop
+      #     would cause memory leaks without such a limit.
+      def errors_tracker
+        coordinator.errors_tracker
+      end
     end
   end
 end

data/lib/karafka/pro/connection/manager.rb

@@ -114,10 +114,15 @@ module Karafka
           once(:quiet!) { active_listeners.each(&:quiet!) }
 
           # If we are in the process of moving to quiet state, we need to check it.
-          if Karafka::App.quieting? && active_listeners.all?(&:quiet?)
+          if Karafka::App.quieting?
+            # If we are quieting but not all active listeners are quiet we need to wait for all of
+            # them to reach the quiet state
+            return unless active_listeners.all?(&:quiet?)
+
             once(:quieted!) { Karafka::App.quieted! }
           end
 
+          # Do nothing if we moved to quiet state and want to be in it
           return if Karafka::App.quiet?
 
           # Since separate subscription groups are subscribed to different topics, there is no risk

data/lib/karafka/pro/processing/coordinator.rb

@@ -21,13 +21,14 @@ module Karafka
 
         def_delegators :@collapser, :collapsed?, :collapse_until!
 
-        attr_reader :filter, :virtual_offset_manager, :shared_mutex
+        attr_reader :filter, :virtual_offset_manager, :shared_mutex, :errors_tracker
 
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
           super
 
           @executed = []
+          @errors_tracker = Coordinators::ErrorsTracker.new
           @flow_mutex = Mutex.new
           # Lock for user code synchronization
           # We do not want to mix coordinator lock with the user lock not to create cases where
@@ -36,11 +37,11 @@ module Karafka
           # framework and can be used for user-facing locking
           @shared_mutex = Mutex.new
           @collapser = Collapser.new
-          @filter = FiltersApplier.new(self)
+          @filter = Coordinators::FiltersApplier.new(self)
 
           return unless topic.virtual_partitions?
 
-          @virtual_offset_manager = VirtualOffsetManager.new(
+          @virtual_offset_manager = Coordinators::VirtualOffsetManager.new(
             topic.name,
             partition,
             topic.virtual_partitions.offset_metadata_strategy
@@ -64,6 +65,14 @@ module Karafka
 
           @filter.apply!(messages)
 
+          # Do not clear coordinator errors storage when we are retrying, so we can reference the
+          # errors that have happened during recovery. This can be useful for implementing custom
+          # flows. There can be more errors than one when running with virtual partitions so we
+          # need to make sure we collect them all. Under collapse when we reference a given
+          # consumer we should be able to get all the errors and not just first/last.
+          #
+          # @note We use zero as the attempt mark because we are not "yet" in the attempt 1
+          @errors_tracker.clear if attempt.zero?
           @executed.clear
 
           # We keep the old processed offsets until the collapsing is done and regular processing
@@ -79,6 +88,7 @@ module Karafka
         # @param error [StandardError] error from the failure
         def failure!(consumer, error)
           super
+          @errors_tracker << error
           collapse_until!(@last_message.offset + 1)
         end
 

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for Pro coordinator related sub-components
+      module Coordinators
+        # Object used to track errors in between executions to be able to build error-type based
+        # recovery flows.
+        class ErrorsTracker
+          include Enumerable
+
+          # Max errors we keep in memory.
+          # We do not want to keep more because for DLQ-less this would cause memory-leaks.
+          STORAGE_LIMIT = 100
+
+          private_constant :STORAGE_LIMIT
+
+          def initialize
+            @errors = []
+          end
+
+          # Clears all the errors
+          def clear
+            @errors.clear
+          end
+
+          # @param error [StandardError] adds the error to the tracker
+          def <<(error)
+            @errors.shift if @errors.size >= STORAGE_LIMIT
+            @errors << error
+          end
+
+          # @return [Boolean] is the error tracker empty
+          def empty?
+            @errors.empty?
+          end
+
+          # @return [Integer] number of elements
+          def size
+            count
+          end
+
+          # @return [StandardError, nil] last error that occurred or nil if no errors
+          def last
+            @errors.last
+          end
+
+          # Iterates over errors
+          # @param block [Proc] code we want to run on each error
+          def each(&block)
+            @errors.each(&block)
+          end
+
+          # @return [Array<StandardError>] array with all the errors that occurred
+          def all
+            @errors
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Coordinators
+        # Applier for all filters we want to have. Whether related to limiting messages based
+        # on the payload or any other things.
+        #
+        # From the outside world perspective, this encapsulates all the filters.
+        # This means that this is the API we expose as a single filter, allowing us to control
+        # the filtering via many filters easily.
+        class FiltersApplier
+          # @return [Array] registered filters array. Useful if we want to inject internal context
+          #   aware filters.
+          attr_reader :filters
+
+          # @param coordinator [Pro::Coordinator] pro coordinator
+          def initialize(coordinator)
+            # Builds filters out of their factories
+            # We build it that way (providing topic and partition) because there may be a case
+            # where someone wants to have a specific logic that is per topic or partition. Like for
+            # example a case where there is a cache bypassing revocations for topic partition.
+            #
+            # We provide full Karafka routing topic here and not the name only, in case the filter
+            # would be customized based on other topic settings (like VPs, etc)
+            #
+            # This setup allows for biggest flexibility also because topic object holds the
+            # reference to the subscription group and consumer group
+            @filters = coordinator.topic.filtering.factories.map do |factory|
+              factory.call(coordinator.topic, coordinator.partition)
+            end
+          end
+
+          # @param messages [Array<Karafka::Messages::Message>] array with messages from the
+          #   partition
+          def apply!(messages)
+            return unless active?
+
+            @filters.each { |filter| filter.apply!(messages) }
+          end
+
+          # @return [Boolean] did we filter out any messages during filtering run
+          def applied?
+            return false unless active?
+
+            !applied.empty?
+          end
+
+          # @return [Symbol] consumer post-filtering action that should be taken
+          def action
+            return :skip unless applied?
+
+            # The highest priority is on a potential backoff from any of the filters because it is
+            # the less risky (delay and continue later)
+            return :pause if applied.any? { |filter| filter.action == :pause }
+
+            # If none of the filters wanted to pause, we can check for any that would want to seek
+            # and if there is any, we can go with this strategy
+            return :seek if applied.any? { |filter| filter.action == :seek }
+
+            :skip
+          end
+
+          # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
+          #   filters to satisfy all of them.
+          def timeout
+            applied.map(&:timeout).compact.min || 0
+          end
+
+          # The first message we do need to get next time we poll. We use the minimum not to jump
+          # accidentally by over any.
+          # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+          # @note Cursor message can also return the offset in the time format
+          def cursor
+            return nil unless active?
+
+            applied.map(&:cursor).compact.min_by(&:offset)
+          end
+
+          private
+
+          # @return [Boolean] is filtering active
+          def active?
+            !@filters.empty?
+          end
+
+          # @return [Array<Object>] filters that applied any sort of messages limiting
+          def applied
+            @filters.select(&:applied?)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Coordinators
+        # Manager that keeps track of our offsets with the virtualization layer that are local
+        # to given partition assignment. It allows for easier offset management for virtual
+        # virtual partition cases as it provides us ability to mark as consumed and move the
+        # real offset behind as expected.
+        #
+        # @note We still use the regular coordinator "real" offset management as we want to have
+        #   them as separated as possible because the real seek offset management is also used for
+        #   pausing, filtering and others and should not be impacted by the virtual one
+        #
+        # @note This manager is **not** thread-safe by itself. It should operate from coordinator
+        #   locked locations.
+        class VirtualOffsetManager
+          attr_reader :groups
+
+          # @param topic [String]
+          # @param partition [Integer]
+          # @param offset_metadata_strategy [Symbol] what metadata should we select. That is,
+          #   should we use the most recent or one picked from the offset that is going to be
+          #   committed
+          #
+          # @note We need topic and partition because we use a seek message (virtual) for real
+          #   offset management. We could keep real message reference but this can be memory
+          #   consuming and not worth it.
+          def initialize(topic, partition, offset_metadata_strategy)
+            @topic = topic
+            @partition = partition
+            @groups = []
+            @marked = {}
+            @offsets_metadata = {}
+            @real_offset = -1
+            @offset_metadata_strategy = offset_metadata_strategy
+            @current_offset_metadata = nil
+          end
+
+          # Clears the manager for a next collective operation
+          def clear
+            @groups.clear
+            @offsets_metadata.clear
+            @current_offset_metadata = nil
+            @marked.clear
+            @real_offset = -1
+          end
+
+          # Registers an offset group coming from one virtual consumer. In order to move the real
+          # underlying offset accordingly, we need to make sure to track the virtual consumers
+          # offsets groups independently and only materialize the end result.
+          #
+          # @param offsets_group [Array<Integer>] offsets from one virtual consumer
+          def register(offsets_group)
+            @groups << offsets_group
+
+            offsets_group.each { |offset| @marked[offset] = false }
+          end
+
+          # Marks given message as marked (virtually consumed).
+          # We mark given message offset and other earlier offsets from the same group as done
+          # and we can refresh our real offset representation based on that as it might have
+          # changed to a newer real offset.
+          # @param message [Karafka::Messages::Message] message coming from VP we want to mark
+          # @param offset_metadata [String, nil] offset metadata. `nil` if none
+          def mark(message, offset_metadata)
+            offset = message.offset
+
+            # Store metadata when we materialize the most stable offset
+            @offsets_metadata[offset] = offset_metadata
+            @current_offset_metadata = offset_metadata
+
+            group = @groups.find { |reg_group| reg_group.include?(offset) }
+
+            # This case can happen when someone uses MoM and wants to mark message from a previous
+            # batch as consumed. We can add it, since the real offset refresh will point to it
+            unless group
+              group = [offset]
+              @groups << group
+            end
+
+            position = group.index(offset)
+
+            # Mark all previous messages from the same group also as virtually consumed
+            group[0..position].each do |markable_offset|
+              # Set previous messages metadata offset as the offset of higher one for overwrites
+              # unless a different metadata were set explicitely
+              @offsets_metadata[markable_offset] ||= offset_metadata
+              @marked[markable_offset] = true
+            end
+
+            # Recompute the real offset representation
+            materialize_real_offset
+          end
+
+          # Mark all from all groups including the `message`.
+          # Useful when operating in a collapsed state for marking
+          # @param message [Karafka::Messages::Message]
+          # @param offset_metadata [String, nil]
+          def mark_until(message, offset_metadata)
+            mark(message, offset_metadata)
+
+            @groups.each do |group|
+              group.each do |offset|
+                next if offset > message.offset
+
+                @offsets_metadata[offset] = offset_metadata
+                @marked[offset] = true
+              end
+            end
+
+            materialize_real_offset
+          end
+
+          # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
+          def marked
+            @marked.select { |_, status| status }.map(&:first).sort
+          end
+
+          # Is there a real offset we can mark as consumed
+          # @return [Boolean]
+          def markable?
+            !@real_offset.negative?
+          end
+
+          # @return [Array<Messages::Seek, String>] markable message for real offset marking and
+          #   its associated metadata
+          def markable
+            raise Errors::InvalidRealOffsetUsageError unless markable?
+
+            offset_metadata = case @offset_metadata_strategy
+                              when :exact
+                                @offsets_metadata.fetch(@real_offset)
+                              when :current
+                                @current_offset_metadata
+                              else
+                                raise Errors::UnsupportedCaseError, @offset_metadata_strategy
+                              end
+
+            [
+              Messages::Seek.new(
+                @topic,
+                @partition,
+                @real_offset
+              ),
+              offset_metadata
+            ]
+          end
+
+          private
+
+          # Recomputes the biggest possible real offset we can have.
+          # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+          # consumed because this will be the collective offset.
+          def materialize_real_offset
+            @marked.to_a.sort_by(&:first).each do |offset, marked|
+              break unless marked
+
+              @real_offset = offset
+            end
+
+            @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
+          end
+        end
+      end
+    end
+  end
+end