karafka 2.0.14 → 2.0.16

data/lib/karafka/time_trackers/pause.rb

@@ -10,7 +10,7 @@ module Karafka
     #   code here, as this is not a frequently used tracker. It is active only once per batch in
     #   case of long-running-jobs and upon errors.
     class Pause < Base
-      attr_reader :count
+      attr_reader :attempt
 
       # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
       # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
@@ -19,26 +19,29 @@ module Karafka
       #   timeout value
       # @return [Karafka::TimeTrackers::Pause]
       # @example
-      #   pause = Karafka::TimeTrackers::Pause.new(timeout: 1000)
+      #   options = { timeout: 1000, max_timeout: 1000, exponential_backoff: false }
+      #   pause = Karafka::TimeTrackers::Pause.new(**options)
       #   pause.expired? #=> true
       #   pause.paused? #=> false
       #   pause.pause
+      #   pause.increment
       #   sleep(1.1)
       #   pause.paused? #=> true
       #   pause.expired? #=> true
-      #   pause.count #=> 1
+      #   pause.attempt #=> 1
       #   pause.pause
-      #   pause.count #=> 1
+      #   pause.increment
+      #   pause.attempt #=> 2
       #   pause.paused? #=> true
       #   pause.expired? #=> false
       #   pause.resume
-      #   pause.count #=> 2
+      #   pause.attempt #=> 2
       #   pause.paused? #=> false
       #   pause.reset
-      #   pause.count #=> 0
+      #   pause.attempt #=> 0
       def initialize(timeout:, max_timeout:, exponential_backoff:)
         @started_at = nil
-        @count = 0
+        @attempt = 0
         @timeout = timeout
         @max_timeout = max_timeout
         @exponential_backoff = exponential_backoff
@@ -47,7 +50,7 @@ module Karafka
       end
 
       # Pauses the processing from now till the end of the interval (backoff or non-backoff)
-      # and records the count.
+      # and records the attempt.
       # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
       # @note Providing this value can be useful when we explicitly want to pause for a certain
       #   period of time, outside of any regular pausing logic
@@ -55,7 +58,13 @@ module Karafka
         @mutex.synchronize do
           @started_at = now
           @ends_at = @started_at + timeout
-          @count += 1
+        end
+      end
+
+      # Increments the number of attempt by 1
+      def increment
+        @mutex.synchronize do
+          @attempt += 1
         end
       end
 
@@ -88,10 +97,10 @@ module Karafka
         end
       end
 
-      # Resets the pause counter.
+      # Resets the pause attempt count.
       def reset
         @mutex.synchronize do
-          @count = 0
+          @attempt = 0
         end
       end
 
@@ -100,7 +109,7 @@ module Karafka
       # Computers the exponential backoff
       # @return [Integer] backoff in milliseconds
       def backoff_interval
-        backoff_factor = @exponential_backoff ? 2**@count : 1
+        backoff_factor = @exponential_backoff ? 2**@attempt : 1
 
         timeout = backoff_factor * @timeout
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.14'
+  VERSION = '2.0.16'
 end

data/lib/karafka.rb

@@ -86,17 +86,13 @@ end
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
-
-begin
-  require 'karafka-license'
-rescue LoadError
-  # Do not load pro components if we cannot load the license
-  # This is a preliminary check so autoload works as expected
-  # Later on the licenser will make sure to setup all the needed components anyhow
-  loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
-end
-
+# Do not load pro components as they will be loaded if needed and allowed
+loader.ignore(Karafka.core_root.join('pro/'))
 # Do not load vendors instrumentation components. Those need to be required manually if needed
-loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
+loader.ignore(Karafka.core_root.join('instrumentation/vendors'))
 loader.setup
 loader.eager_load
+
+# This will load features but since Pro are not loaded automatically, they will not be visible
+# nor included here
+::Karafka::Routing::Features::Base.load_all

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.14
+  version: 2.0.16
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-10-16 00:00:00.000000000 Z
+date: 2022-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -150,6 +150,7 @@ files:
 - bin/create_token
 - bin/integrations
 - bin/karafka
+- bin/rspecs
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
@@ -165,7 +166,6 @@ files:
 - lib/karafka/active_job/dispatcher.rb
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
-- lib/karafka/active_job/routing/extensions.rb
 - lib/karafka/admin.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
@@ -186,8 +186,8 @@ files:
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
-- lib/karafka/contracts/consumer_group_topic.rb
 - lib/karafka/contracts/server_cli_options.rb
+- lib/karafka/contracts/topic.rb
 - lib/karafka/embedded.rb
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
@@ -204,6 +204,7 @@ files:
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
 - lib/karafka/instrumentation/vendors/datadog/listener.rb
+- lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -220,9 +221,6 @@ files:
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
 - lib/karafka/pro/base_consumer.rb
-- lib/karafka/pro/contracts/base.rb
-- lib/karafka/pro/contracts/consumer_group.rb
-- lib/karafka/pro/contracts/consumer_group_topic.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/coordinator.rb
@@ -230,8 +228,35 @@ files:
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
-- lib/karafka/pro/routing/builder_extensions.rb
-- lib/karafka/pro/routing/topic_extensions.rb
+- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
+- lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/base.rb
+- lib/karafka/pro/processing/strategies/default.rb
+- lib/karafka/pro/processing/strategies/dlq.rb
+- lib/karafka/pro/processing/strategies/dlq_lrj.rb
+- lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq_mom.rb
+- lib/karafka/pro/processing/strategies/lrj.rb
+- lib/karafka/pro/processing/strategies/lrj_mom.rb
+- lib/karafka/pro/processing/strategies/lrj_vp.rb
+- lib/karafka/pro/processing/strategies/mom.rb
+- lib/karafka/pro/processing/strategies/vp.rb
+- lib/karafka/pro/processing/strategy_selector.rb
+- lib/karafka/pro/routing/features/base.rb
+- lib/karafka/pro/routing/features/dead_letter_queue.rb
+- lib/karafka/pro/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/pro/routing/features/long_running_job.rb
+- lib/karafka/pro/routing/features/long_running_job/config.rb
+- lib/karafka/pro/routing/features/long_running_job/contract.rb
+- lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/virtual_partitions.rb
+- lib/karafka/pro/routing/features/virtual_partitions/config.rb
+- lib/karafka/pro/routing/features/virtual_partitions/contract.rb
+- lib/karafka/pro/routing/features/virtual_partitions/topic.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
 - lib/karafka/processing/coordinators_buffer.rb
@@ -246,12 +271,35 @@ files:
 - lib/karafka/processing/partitioner.rb
 - lib/karafka/processing/result.rb
 - lib/karafka/processing/scheduler.rb
+- lib/karafka/processing/strategies/aj_dlq_mom.rb
+- lib/karafka/processing/strategies/aj_mom.rb
+- lib/karafka/processing/strategies/base.rb
+- lib/karafka/processing/strategies/default.rb
+- lib/karafka/processing/strategies/dlq.rb
+- lib/karafka/processing/strategies/dlq_mom.rb
+- lib/karafka/processing/strategies/mom.rb
+- lib/karafka/processing/strategy_selector.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
 - lib/karafka/routing/consumer_mapper.rb
+- lib/karafka/routing/features/active_job.rb
+- lib/karafka/routing/features/active_job/builder.rb
+- lib/karafka/routing/features/active_job/config.rb
+- lib/karafka/routing/features/active_job/contract.rb
+- lib/karafka/routing/features/active_job/topic.rb
+- lib/karafka/routing/features/base.rb
+- lib/karafka/routing/features/base/expander.rb
+- lib/karafka/routing/features/dead_letter_queue.rb
+- lib/karafka/routing/features/dead_letter_queue/config.rb
+- lib/karafka/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/routing/features/dead_letter_queue/topic.rb
+- lib/karafka/routing/features/manual_offset_management.rb
+- lib/karafka/routing/features/manual_offset_management/config.rb
+- lib/karafka/routing/features/manual_offset_management/contract.rb
+- lib/karafka/routing/features/manual_offset_management/topic.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/subscription_group.rb

data/lib/karafka/active_job/routing/extensions.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # ActiveJob related Karafka stuff
-  module ActiveJob
-    # Karafka routing ActiveJob related components
-    module Routing
-      # Routing extensions for ActiveJob
-      module Extensions
-        # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
-        # the consumer class
-        # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-        # @param block [Proc] block that we can use for some extra configuration
-        def active_job_topic(name, &block)
-          topic(name) do
-            consumer App.config.internal.active_job.consumer_class
-
-            next unless block
-
-            instance_eval(&block)
-
-            target.tags << :active_job
-
-            # This is handled by our custom ActiveJob consumer
-            # Without this, default behaviour would cause messages to skip upon shutdown as the
-            # offset would be committed for the last message
-            manual_offset_management true
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/contracts/consumer_group.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Contracts
-      # Contract for validating correct Pro components setup on a consumer group and topic levels
-      class ConsumerGroup < Base
-        virtual do |data, errors|
-          next unless errors.empty?
-          next unless data.key?(:topics)
-
-          fetched_errors = []
-
-          data.fetch(:topics).each do |topic|
-            ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
-              fetched_errors << [[topic, key].flatten, value]
-            end
-          end
-
-          fetched_errors
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Contracts
-      # Contract for validating correct Pro components setup on a topic levels
-      class ConsumerGroupTopic < Base
-        configure do |config|
-          config.error_messages = YAML.safe_load(
-            File.read(
-              File.join(Karafka.gem_root, 'config', 'errors.yml')
-            )
-          ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
-        end
-
-        nested(:virtual_partitions) do
-          required(:active) { |val| [true, false].include?(val) }
-          required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
-          required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
-        end
-
-        virtual do |data, errors|
-          next unless errors.empty?
-          next if data[:consumer] < Karafka::Pro::BaseConsumer
-
-          [[%i[consumer], :consumer_format]]
-        end
-
-        # When virtual partitions are defined, partitioner needs to respond to `#call` and it
-        # cannot be nil
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          virtual_partitions = data[:virtual_partitions]
-
-          next unless virtual_partitions[:active]
-          next if virtual_partitions[:partitioner].respond_to?(:call)
-
-          [[%i[virtual_partitions partitioner], :respond_to_call]]
-        end
-
-        # Make sure that manual offset management is not used together with Virtual Partitions
-        # This would not make any sense as there would be edge cases related to skipping
-        # messages even if there were errors.
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          virtual_partitions = data[:virtual_partitions]
-          manual_offset_management = data[:manual_offset_management]
-
-          next unless virtual_partitions[:active]
-          next unless manual_offset_management
-          next if data[:tags].include?(:active_job)
-
-          [[%i[manual_offset_management], :not_with_virtual_partitions]]
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Pro routing components
-    module Routing
-      # Routing extensions that allow to configure some extra PRO routing options
-      module TopicExtensions
-        # Internal representation of the virtual partitions settings and configuration
-        # This allows us to abstract away things in a nice manner
-        #
-        # For features with more options than just on/off we use this approach as it simplifies
-        # the code. We do not use it for all not to create unneeded complexity
-        VirtualPartitions = Struct.new(
-          :active,
-          :partitioner,
-          :max_partitions,
-          keyword_init: true
-        ) { alias_method :active?, :active }
-
-        class << self
-          # @param base [Class] class we extend
-          def prepended(base)
-            base.attr_accessor :long_running_job
-          end
-        end
-
-        # @param max_partitions [Integer] max number of virtual partitions that can come out of the
-        #   single distribution flow. When set to more than the Karafka threading, will create
-        #   more work than workers. When less, can ensure we have spare resources to process other
-        #   things in parallel.
-        # @param partitioner [nil, #call] nil or callable partitioner
-        # @return [VirtualPartitions] method that allows to set the virtual partitions details
-        #   during the routing configuration and then allows to retrieve it
-        def virtual_partitions(
-          max_partitions: Karafka::App.config.concurrency,
-          partitioner: nil
-        )
-          @virtual_partitions ||= VirtualPartitions.new(
-            active: !partitioner.nil?,
-            max_partitions: max_partitions,
-            partitioner: partitioner
-          )
-        end
-
-        # @return [Boolean] are virtual partitions enabled for given topic
-        def virtual_partitions?
-          virtual_partitions.active?
-        end
-
-        # @return [Boolean] is a given job on a topic a long-running one
-        def long_running_job?
-          @long_running_job || false
-        end
-
-        # @return [Hash] hash with topic details and the extensions details
-        def to_h
-          super.merge(
-            virtual_partitions: virtual_partitions.to_h
-          )
-        end
-      end
-    end
-  end
-end