karafka 2.3.4 → 2.4.0.beta1

data/lib/karafka/processing/executors_buffer.rb

@@ -5,6 +5,10 @@ module Karafka
     # Buffer for executors of a given subscription group. It wraps around the concept of building
     # and caching them, so we can re-use them instead of creating new each time.
     class ExecutorsBuffer
+      include Helpers::ConfigImporter.new(
+        executor_class: %i[internal processing executor_class]
+      )
+
       # @param client [Connection::Client]
       # @param subscription_group [Routing::SubscriptionGroup]
       # @return [ExecutorsBuffer]
@@ -13,7 +17,6 @@ module Karafka
         @client = client
         # We need two layers here to keep track of topics, partitions and processing groups
         @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
-        @executor_class = Karafka::App.config.internal.processing.executor_class
       end
 
       # Finds or creates an executor based on the provided details
@@ -24,7 +27,7 @@ module Karafka
       # @param coordinator [Karafka::Processing::Coordinator]
       # @return [Executor, Pro::Processing::Executor] consumer executor
       def find_or_create(topic, partition, parallel_key, coordinator)
-        @buffer[topic][partition][parallel_key] ||= @executor_class.new(
+        @buffer[topic][partition][parallel_key] ||= executor_class.new(
           @subscription_group.id,
           @client,
           coordinator

data/lib/karafka/processing/jobs_queue.rb

@@ -13,6 +13,11 @@ module Karafka
     # @note This job queue also keeps track / understands number of busy workers. This is because
     #   we use a single workers poll that can have granular scheduling.
     class JobsQueue
+      include Helpers::ConfigImporter.new(
+        concurrency: %i[concurrency],
+        tick_interval: %i[internal tick_interval]
+      )
+
       # @return [Karafka::Processing::JobsQueue]
       def initialize
         @queue = Queue.new
@@ -24,8 +29,6 @@ module Karafka
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
         @semaphores = {}
-        @concurrency = Karafka::App.config.concurrency
-        @tick_interval = ::Karafka::App.config.internal.tick_interval
         @in_processing = Hash.new { |h, k| h[k] = [] }
         @statistics = { busy: 0, enqueued: 0 }
 
@@ -67,7 +70,7 @@ module Karafka
           # Assume that moving to queue means being picked up immediately not to create stats
           # race conditions because of pop overhead. If there are workers available, we assume
           # work is going to be handled as we never reject enqueued jobs
-          if @statistics[:busy] < @concurrency
+          if @statistics[:busy] < concurrency
             @statistics[:busy] += 1
           else
             # If system is fully loaded, it means this job is indeed enqueued
@@ -155,12 +158,14 @@ module Karafka
       #   the work to be finished.
       # @note This method is blocking.
       def wait(group_id)
+        interval_in_seconds = tick_interval / 1_000.0
+
         # Go doing other things while we cannot process and wait for anyone to finish their work
         # and re-check the wait status
         while wait?(group_id)
           yield if block_given?
 
-          @semaphores.fetch(group_id).pop(timeout: @tick_interval / 1_000.0)
+          @semaphores.fetch(group_id).pop(timeout: interval_in_seconds)
         end
       end
 

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -34,7 +34,7 @@ module Karafka
             # We can commit the offset here because we know that we skip it "forever" and
             # since AJ consumer commits the offset after each job, we also know that the
             # previous job was successful
-            mark_as_consumed(skippable_message)
+            mark_dispatched_to_dlq(skippable_message)
             pause(coordinator.seek_offset, nil, false)
           end
         end

data/lib/karafka/processing/strategies/default.rb

@@ -120,7 +120,7 @@ module Karafka
           raise e
         ensure
           # We need to decrease number of jobs that this coordinator coordinates as it has finished
-          coordinator.decrement
+          coordinator.decrement(:consume)
         end
 
         # Standard flow marks work as consumed and moves on if everything went ok.
@@ -147,6 +147,8 @@ module Karafka
         # Code that should run on idle runs without messages available
         def handle_idle
           nil
+        ensure
+          coordinator.decrement(:idle)
         end
 
         # We need to always un-pause the processing in case we have lost a given partition.
@@ -161,6 +163,8 @@ module Karafka
           Karafka.monitor.instrument('consumer.revoked', caller: self) do
             revoked
           end
+        ensure
+          coordinator.decrement(:revoked)
         end
 
         # Runs the shutdown code
@@ -169,6 +173,8 @@ module Karafka
           Karafka.monitor.instrument('consumer.shutdown', caller: self) do
             shutdown
           end
+        ensure
+          coordinator.decrement(:shutdown)
         end
       end
     end

data/lib/karafka/processing/strategies/dlq.rb

@@ -76,7 +76,7 @@ module Karafka
             dispatch_to_dlq(skippable_message)
 
             # We mark the broken message as consumed and move on
-            mark_as_consumed(skippable_message)
+            mark_dispatched_to_dlq(skippable_message)
 
             return if revoked?
 
@@ -106,7 +106,8 @@ module Karafka
         # @param skippable_message [Karafka::Messages::Message] message we are skipping that also
         #   should go to the dlq topic
         def dispatch_to_dlq(skippable_message)
-          producer.produce_async(
+          producer.public_send(
+            topic.dead_letter_queue.dispatch_method,
             topic: topic.dead_letter_queue.topic,
             payload: skippable_message.raw_payload
           )
@@ -118,6 +119,20 @@ module Karafka
             message: skippable_message
           )
         end
+
+        # Marks message that went to DLQ (if applicable) based on the requested method
+        # @param skippable_message [Karafka::Messages::Message]
+        def mark_dispatched_to_dlq(skippable_message)
+          case topic.dead_letter_queue.marking_method
+          when :mark_as_consumed
+            mark_as_consumed(skippable_message)
+          when :mark_as_consumed!
+            mark_as_consumed!(skippable_message)
+          else
+            # This should never happen. Bug if encountered. Please report
+            raise Karafka::Errors::UnsupportedCaseError
+          end
+        end
       end
     end
   end

data/lib/karafka/processing/workers_batch.rb

@@ -5,11 +5,14 @@ module Karafka
     # Abstraction layer around workers batch.
     class WorkersBatch
       include Enumerable
+      include Helpers::ConfigImporter.new(
+        concurrency: %i[concurrency]
+      )
 
       # @param jobs_queue [JobsQueue]
       # @return [WorkersBatch]
       def initialize(jobs_queue)
-        @batch = Array.new(App.config.concurrency) { Processing::Worker.new(jobs_queue) }
+        @batch = Array.new(concurrency) { Processing::Worker.new(jobs_queue) }
       end
 
       # Iterates over available workers and yields each worker

data/lib/karafka/routing/builder.rb

@@ -14,6 +14,10 @@ module Karafka
     #     end
     #   end
     class Builder < Array
+      include Helpers::ConfigImporter.new(
+        default_group_id: %i[group_id]
+      )
+
       # Empty default per-topic config
       EMPTY_DEFAULTS = ->(_) {}.freeze
 
@@ -116,7 +120,7 @@ module Karafka
         **args,
         &block
       )
-        consumer_group('app') do
+        consumer_group(default_group_id) do
           target.public_send(
             :subscription_group=,
             subscription_group_name.to_s,
@@ -132,7 +136,7 @@ module Karafka
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
       # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
-        consumer_group('app') do
+        consumer_group(default_group_id) do
           topic(topic_name, &block)
         end
       end

data/lib/karafka/routing/consumer_group.rb

@@ -22,7 +22,8 @@ module Karafka
       #   kafka and don't understand the concept of consumer groups.
       def initialize(name)
         @name = name.to_s
-        @id = config.consumer_mapper.call(name)
+        # This used to be different when consumer mappers existed but now it is the same
+        @id = @name
         @topics = Topics.new([])
         # Initialize the subscription group so there's always a value for it, since even if not
         # defined directly, a subscription group will be created

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -17,6 +17,11 @@ module Karafka
           :transactional,
           # Strategy to apply (if strategies supported)
           :strategy,
+          # Should we use `#produce_sync` or `#produce_async`
+          :dispatch_method,
+          # Should we use `#mark_as_consumed` or `#mark_as_consumed!` (in flows that mark)
+          :marking_method,
+          # Initialize with kwargs
           keyword_init: true
         ) do
           alias_method :active?, :active

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -21,6 +21,14 @@ module Karafka
               required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
               required(:transactional) { |val| [true, false].include?(val) }
+
+              required(:dispatch_method) do |val|
+                %i[produce_async produce_sync].include?(val)
+              end
+
+              required(:marking_method) do |val|
+                %i[mark_as_consumed mark_as_consumed!].include?(val)
+              end
             end
 
             # Validate topic name only if dlq is active

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -18,19 +18,27 @@ module Karafka
           #   in a retry flow to reset the errors counter
           # @param transactional [Boolean] if applicable, should transaction be used to move
           #   given message to the dead-letter topic and mark it as consumed.
+          # @param dispatch_method [Symbol] `:produce_async` or `:produce_sync`. Describes
+          #   whether dispatch on dlq should be sync or async (async by default)
+          # @param marking_method [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Describes
+          #   whether marking on DLQ should be async or sync (async by default)
           # @return [Config] defined config
           def dead_letter_queue(
             max_retries: DEFAULT_MAX_RETRIES,
             topic: nil,
             independent: false,
-            transactional: true
+            transactional: true,
+            dispatch_method: :produce_async,
+            marking_method: :mark_as_consumed
           )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
               max_retries: max_retries,
               topic: topic,
               independent: independent,
-              transactional: transactional
+              transactional: transactional,
+              dispatch_method: dispatch_method,
+              marking_method: marking_method
             )
           end
 

data/lib/karafka/routing/features/deserializers/config.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Deserializers < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          :payload,
+          :key,
+          :headers,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/deserializers/contracts/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Deserializers < Base
+        # This feature validation contracts
+        module Contracts
+          # Basic validation of the Kafka expected config details
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :deserializers do
+              # Always enabled
+              required(:active) { |val| val == true }
+              required(:payload) { |val| val.respond_to?(:call) }
+              required(:headers) { |val| val.respond_to?(:call) }
+              required(:key) { |val| val.respond_to?(:call) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Deserializers for all the message details (payload, headers, key)
+      class Deserializers < Base
+        # Routing topic deserializers API. It allows to configure deserializers for various
+        # components of each message.
+        module Topic
+          # Allows for setting all the deserializers with standard defaults
+          # @param payload [Object] Deserializer for the message payload
+          # @param key [Object] deserializer for the message key
+          # @param headers [Object] deserializer for the message headers
+          def deserializers(
+            payload: ::Karafka::Deserializers::Payload.new,
+            key: ::Karafka::Deserializers::Key.new,
+            headers: ::Karafka::Deserializers::Headers.new
+          )
+            @deserializers ||= Config.new(
+              active: true,
+              payload: payload,
+              key: key,
+              headers: headers
+            )
+          end
+
+          # Supports pre 2.4 format where only payload deserializer could be defined. We do not
+          # retire this format because it is not bad when users do not do anything advanced with
+          # key or headers
+          # @param payload [Object] payload deserializer
+          def deserializer(payload)
+            deserializers(payload: payload)
+          end
+
+          # @return [Boolean] Deserializers are always active
+          def deserializers?
+            deserializers.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              deserializers: deserializers.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/deserializers.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Namespace for feature allowing to configure deserializers for payload, key and headers
+      class Deserializers < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/proxy.rb

@@ -18,22 +18,17 @@ module Karafka
         instance_eval(&defaults) if defaults
       end
 
-      # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way
-      arg_forwarding = RUBY_VERSION < '3.0' ? '*args, &block' : '...'
-
-      class_eval <<~RUBY, __FILE__, __LINE__ + 1
-        # Translates the no "=" DSL of routing into elements assignments on target
-        # @param method_name [Symbol] name of the missing method
-        def method_missing(method_name, #{arg_forwarding})
-          return super unless respond_to_missing?(method_name)
+      # Translates the no "=" DSL of routing into elements assignments on target
+      # @param method_name [Symbol] name of the missing method
+      def method_missing(method_name, ...)
+        return super unless respond_to_missing?(method_name)
 
-          if @target.respond_to?(:"\#{method_name}=")
-            @target.public_send(:"\#{method_name}=", #{arg_forwarding})
-          else
-            @target.public_send(method_name, #{arg_forwarding})
-          end
+        if @target.respond_to?(:"#{method_name}=")
+          @target.public_send(:"#{method_name}=", ...)
+        else
+          @target.public_send(method_name, ...)
         end
-      RUBY
+      end
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method

data/lib/karafka/routing/router.rb

@@ -23,7 +23,7 @@ module Karafka
       end
 
       # Finds the topic by name (in any consumer group) and if not present, will built a new
-      # representation of the topic with the defaults and default deserializer.
+      # representation of the topic with the defaults and default deserializers.
       #
       # This is used in places where we may operate on topics that are not part of the routing
       # but we want to do something on them (display data, iterate over, etc)
@@ -33,7 +33,16 @@ module Karafka
       # @note Please note, that in case of a new topic, it will have a newly built consumer group
       #   as well, that is not part of the routing.
       def find_or_initialize_by_name(name)
-        find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
+        existing_topic = find_by(name: name)
+
+        return existing_topic if existing_topic
+
+        virtual_topic = Topic.new(name, ConsumerGroup.new(name))
+
+        Karafka::Routing::Proxy.new(
+          virtual_topic,
+          Karafka::App.config.internal.routing.builder.defaults
+        ).target
       end
 
       module_function :find_by

data/lib/karafka/routing/subscription_group.rb

@@ -76,7 +76,8 @@ module Karafka
         activity_manager.active?(:subscription_groups, name)
       end
 
-      # @return [Array<String>] names of topics to which we should subscribe.
+      # @return [false, Array<String>] names of topics to which we should subscribe or false when
+      #   operating only on direct assignments
       #
       # @note Most of the time it should not include inactive topics but in case of pattern
       #   matching the matcher topics become inactive down the road, hence we filter out so
@@ -85,6 +86,13 @@ module Karafka
         topics.select(&:active?).map(&:subscription_name)
       end
 
+      # @param _consumer [Karafka::Connection::Proxy]
+      # @return [false, Rdkafka::Consumer::TopicPartitionList] List of tpls for direct assignments
+      #   or false for the normal mode
+      def assignments(_consumer)
+        false
+      end
+
       # @return [String] id of the subscription group
       # @note This is an alias for displaying in places where we print the stringified version.
       def to_s

data/lib/karafka/routing/topic.rb

@@ -18,7 +18,6 @@ module Karafka
       # Attributes we can inherit from the root unless they were defined on this level
       INHERITABLE_ATTRIBUTES = %i[
         kafka
-        deserializer
         max_messages
         max_wait_time
         initial_offset

data/lib/karafka/runner.rb

@@ -25,7 +25,7 @@ module Karafka
       # Register all the listeners so they can be started and managed
       @manager.register(listeners)
 
-      workers.each(&:async_call)
+      workers.each_with_index { |worker, i| worker.async_call("karafka.worker##{i}") }
 
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers

data/lib/karafka/setup/config.rb

@@ -64,15 +64,9 @@ module Karafka
       setting :logger, default: ::Karafka::Instrumentation::Logger.new
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
       setting :monitor, default: ::Karafka::Instrumentation::Monitor.new
-      # Mapper used to remap consumer groups ids, so in case users migrate from other tools
-      # or they need to maintain their own internal consumer group naming conventions, they
-      # can easily do it, replacing the default client_id + consumer name pattern concept
-      setting :consumer_mapper, default: Routing::ConsumerMapper.new
       # option [Boolean] should we reload consumers with each incoming batch thus effectively
       # supporting code reload (if someone reloads code) or should we keep the persistence
       setting :consumer_persistence, default: true
-      # Default deserializer for converting incoming data into ruby objects
-      setting :deserializer, default: Karafka::Serialization::Json::Deserializer.new
       # option [String] should we start with the earliest possible offset or latest
       # This will set the `auto.offset.reset` value unless present in the kafka scope
       setting :initial_offset, default: 'earliest'
@@ -100,6 +94,15 @@ module Karafka
       # Disabling this may be needed in scenarios where we do not have control over topics names
       # and/or we work with existing systems where we cannot change topics names.
       setting :strict_topics_namespacing, default: true
+      # option [String] default consumer group name for implicit routing
+      setting :group_id, default: 'app'
+
+      setting :oauth do
+        # option [false, #call] Listener for using oauth bearer. This listener will be able to
+        #   get the client name to decide whether to use a single multi-client token refreshing
+        #   or have separate tokens per instance.
+        setting :token_provider_listener, default: false
+      end
 
       # rdkafka default options
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
@@ -137,11 +140,12 @@ module Karafka
           # involving a consumer instance
           'enable.auto.commit': false,
           # Make sure that topic metadata lookups do not create topics accidentally
-          'allow.auto.create.topics': false
+          'allow.auto.create.topics': false,
+          # Do not store offsets automatically in admin in any way
+          'enable.auto.offset.store': false
         }
 
-        # option [String] default name for the admin consumer group. Please note, that this is a
-        # subject to be remapped by the consumer mapper as any other consumer group in the routes
+        # option [String] default name for the admin consumer group.
         setting :group_id, default: 'karafka_admin'
 
         # option max_wait_time [Integer] We wait only for this amount of time before raising error
@@ -230,6 +234,14 @@ module Karafka
 
           # Settings that are altered by our client proxy layer
           setting :proxy do
+            # commit offsets request
+            setting :commit do
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
             # Committed offsets for given CG query
             setting :committed do
               # timeout for this request. For busy or remote clusters, this should be high enough
@@ -259,6 +271,26 @@ module Karafka
               # How long should we wait before next attempt in case of a failure
               setting :wait_time, default: 1_000
             end
+
+            # Settings for lag request
+            setting :lag do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 10_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
+            # Settings for metadata request
+            setting :metadata do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 10_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
           end
         end
 
@@ -368,10 +400,19 @@ module Karafka
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components
+          oauth_listener = config.oauth.token_provider_listener
+          # We need to subscribe the oauth listener here because we want it to be ready before
+          # any consumer/admin runs
+          Karafka::App.monitor.subscribe(oauth_listener) if oauth_listener
+
           config.producer ||= ::WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
             producer_config.kafka = AttributesMap.producer(config.kafka.dup)
+            # We also propagate same listener to the default producer to make sure, that the
+            # listener for oauth is also automatically used by the producer. That way we don't
+            # have to configure it manually for the default producer
+            producer_config.oauth.token_provider_listener = oauth_listener
             producer_config.logger = config.logger
           end
         end

data/lib/karafka/status.rb

@@ -3,6 +3,11 @@
 module Karafka
   # App status monitor
   class Status
+    include Helpers::ConfigImporter.new(
+      monitor: %i[monitor],
+      conductor: %i[internal connection conductor]
+    )
+
     # Available states and their transitions.
     STATES = {
       initializing: :initialize!,
@@ -60,14 +65,8 @@ module Karafka
             # We skip as during this state we do not have yet a monitor
             return if initializing?
 
-            # We do not set conductor in the initializer because this status object is created
-            # before the configuration kicks in
-            # We need to signal conductor on each state change as those may be relevant to
-            # listeners operations
-            @conductor ||= Karafka::App.config.internal.connection.conductor
-            @conductor.signal
-
-            Karafka.monitor.instrument("app.#{state}")
+            conductor.signal
+            monitor.instrument("app.#{state}", caller: self)
           end
         end
       RUBY