karafka 2.0.15 → 2.0.16

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.15
+  version: 2.0.16
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-10-20 00:00:00.000000000 Z
+date: 2022-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -150,6 +150,7 @@ files:
 - bin/create_token
 - bin/integrations
 - bin/karafka
+- bin/rspecs
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
@@ -165,7 +166,6 @@ files:
 - lib/karafka/active_job/dispatcher.rb
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
-- lib/karafka/active_job/routing/extensions.rb
 - lib/karafka/admin.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
@@ -186,8 +186,8 @@ files:
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
-- lib/karafka/contracts/consumer_group_topic.rb
 - lib/karafka/contracts/server_cli_options.rb
+- lib/karafka/contracts/topic.rb
 - lib/karafka/embedded.rb
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
@@ -204,6 +204,7 @@ files:
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
 - lib/karafka/instrumentation/vendors/datadog/listener.rb
+- lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -220,9 +221,6 @@ files:
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
 - lib/karafka/pro/base_consumer.rb
-- lib/karafka/pro/contracts/base.rb
-- lib/karafka/pro/contracts/consumer_group.rb
-- lib/karafka/pro/contracts/consumer_group_topic.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/coordinator.rb
@@ -230,8 +228,35 @@ files:
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
-- lib/karafka/pro/routing/builder_extensions.rb
-- lib/karafka/pro/routing/topic_extensions.rb
+- lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_dlq_mom.rb
+- lib/karafka/pro/processing/strategies/aj_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/aj_mom.rb
+- lib/karafka/pro/processing/strategies/aj_mom_vp.rb
+- lib/karafka/pro/processing/strategies/base.rb
+- lib/karafka/pro/processing/strategies/default.rb
+- lib/karafka/pro/processing/strategies/dlq.rb
+- lib/karafka/pro/processing/strategies/dlq_lrj.rb
+- lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq_mom.rb
+- lib/karafka/pro/processing/strategies/lrj.rb
+- lib/karafka/pro/processing/strategies/lrj_mom.rb
+- lib/karafka/pro/processing/strategies/lrj_vp.rb
+- lib/karafka/pro/processing/strategies/mom.rb
+- lib/karafka/pro/processing/strategies/vp.rb
+- lib/karafka/pro/processing/strategy_selector.rb
+- lib/karafka/pro/routing/features/base.rb
+- lib/karafka/pro/routing/features/dead_letter_queue.rb
+- lib/karafka/pro/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/pro/routing/features/long_running_job.rb
+- lib/karafka/pro/routing/features/long_running_job/config.rb
+- lib/karafka/pro/routing/features/long_running_job/contract.rb
+- lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/virtual_partitions.rb
+- lib/karafka/pro/routing/features/virtual_partitions/config.rb
+- lib/karafka/pro/routing/features/virtual_partitions/contract.rb
+- lib/karafka/pro/routing/features/virtual_partitions/topic.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
 - lib/karafka/processing/coordinators_buffer.rb
@@ -246,12 +271,35 @@ files:
 - lib/karafka/processing/partitioner.rb
 - lib/karafka/processing/result.rb
 - lib/karafka/processing/scheduler.rb
+- lib/karafka/processing/strategies/aj_dlq_mom.rb
+- lib/karafka/processing/strategies/aj_mom.rb
+- lib/karafka/processing/strategies/base.rb
+- lib/karafka/processing/strategies/default.rb
+- lib/karafka/processing/strategies/dlq.rb
+- lib/karafka/processing/strategies/dlq_mom.rb
+- lib/karafka/processing/strategies/mom.rb
+- lib/karafka/processing/strategy_selector.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
 - lib/karafka/routing/consumer_mapper.rb
+- lib/karafka/routing/features/active_job.rb
+- lib/karafka/routing/features/active_job/builder.rb
+- lib/karafka/routing/features/active_job/config.rb
+- lib/karafka/routing/features/active_job/contract.rb
+- lib/karafka/routing/features/active_job/topic.rb
+- lib/karafka/routing/features/base.rb
+- lib/karafka/routing/features/base/expander.rb
+- lib/karafka/routing/features/dead_letter_queue.rb
+- lib/karafka/routing/features/dead_letter_queue/config.rb
+- lib/karafka/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/routing/features/dead_letter_queue/topic.rb
+- lib/karafka/routing/features/manual_offset_management.rb
+- lib/karafka/routing/features/manual_offset_management/config.rb
+- lib/karafka/routing/features/manual_offset_management/contract.rb
+- lib/karafka/routing/features/manual_offset_management/topic.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/subscription_group.rb

data/lib/karafka/active_job/routing/extensions.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # ActiveJob related Karafka stuff
-  module ActiveJob
-    # Karafka routing ActiveJob related components
-    module Routing
-      # Routing extensions for ActiveJob
-      module Extensions
-        # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
-        # the consumer class
-        # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-        # @param block [Proc] block that we can use for some extra configuration
-        def active_job_topic(name, &block)
-          topic(name) do
-            consumer App.config.internal.active_job.consumer_class
-
-            next unless block
-
-            instance_eval(&block)
-
-            target.tags << :active_job
-
-            # This is handled by our custom ActiveJob consumer
-            # Without this, default behaviour would cause messages to skip upon shutdown as the
-            # offset would be committed for the last message
-            manual_offset_management true
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/contracts/consumer_group.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Contracts
-      # Contract for validating correct Pro components setup on a consumer group and topic levels
-      class ConsumerGroup < Base
-        virtual do |data, errors|
-          next unless errors.empty?
-          next unless data.key?(:topics)
-
-          fetched_errors = []
-
-          data.fetch(:topics).each do |topic|
-            ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
-              fetched_errors << [[topic, key].flatten, value]
-            end
-          end
-
-          fetched_errors
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Contracts
-      # Contract for validating correct Pro components setup on a topic levels
-      class ConsumerGroupTopic < Base
-        configure do |config|
-          config.error_messages = YAML.safe_load(
-            File.read(
-              File.join(Karafka.gem_root, 'config', 'errors.yml')
-            )
-          ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
-        end
-
-        nested(:virtual_partitions) do
-          required(:active) { |val| [true, false].include?(val) }
-          required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
-          required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
-        end
-
-        virtual do |data, errors|
-          next unless errors.empty?
-          next if data[:consumer] < Karafka::Pro::BaseConsumer
-
-          [[%i[consumer], :consumer_format]]
-        end
-
-        # When virtual partitions are defined, partitioner needs to respond to `#call` and it
-        # cannot be nil
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          virtual_partitions = data[:virtual_partitions]
-
-          next unless virtual_partitions[:active]
-          next if virtual_partitions[:partitioner].respond_to?(:call)
-
-          [[%i[virtual_partitions partitioner], :respond_to_call]]
-        end
-
-        # Make sure that manual offset management is not used together with Virtual Partitions
-        # This would not make any sense as there would be edge cases related to skipping
-        # messages even if there were errors.
-        virtual do |data, errors|
-          next unless errors.empty?
-
-          virtual_partitions = data[:virtual_partitions]
-          manual_offset_management = data[:manual_offset_management]
-
-          next unless virtual_partitions[:active]
-          next unless manual_offset_management
-          next if data[:tags].include?(:active_job)
-
-          [[%i[manual_offset_management], :not_with_virtual_partitions]]
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    # Pro routing components
-    module Routing
-      # Routing extensions that allow to configure some extra PRO routing options
-      module TopicExtensions
-        # Internal representation of the virtual partitions settings and configuration
-        # This allows us to abstract away things in a nice manner
-        #
-        # For features with more options than just on/off we use this approach as it simplifies
-        # the code. We do not use it for all not to create unneeded complexity
-        VirtualPartitions = Struct.new(
-          :active,
-          :partitioner,
-          :max_partitions,
-          keyword_init: true
-        ) { alias_method :active?, :active }
-
-        class << self
-          # @param base [Class] class we extend
-          def prepended(base)
-            base.attr_accessor :long_running_job
-          end
-        end
-
-        # @param max_partitions [Integer] max number of virtual partitions that can come out of the
-        #   single distribution flow. When set to more than the Karafka threading, will create
-        #   more work than workers. When less, can ensure we have spare resources to process other
-        #   things in parallel.
-        # @param partitioner [nil, #call] nil or callable partitioner
-        # @return [VirtualPartitions] method that allows to set the virtual partitions details
-        #   during the routing configuration and then allows to retrieve it
-        def virtual_partitions(
-          max_partitions: Karafka::App.config.concurrency,
-          partitioner: nil
-        )
-          @virtual_partitions ||= VirtualPartitions.new(
-            active: !partitioner.nil?,
-            max_partitions: max_partitions,
-            partitioner: partitioner
-          )
-        end
-
-        # @return [Boolean] are virtual partitions enabled for given topic
-        def virtual_partitions?
-          virtual_partitions.active?
-        end
-
-        # @return [Boolean] is a given job on a topic a long-running one
-        def long_running_job?
-          @long_running_job || false
-        end
-
-        # @return [Hash] hash with topic details and the extensions details
-        def to_h
-          super.merge(
-            virtual_partitions: virtual_partitions.to_h
-          )
-        end
-      end
-    end
-  end
-end