karafka 2.0.23 → 2.0.26

data/lib/karafka/pro/encryption.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Out of the box encryption engine for both Karafka and WaterDrop
+    # It uses asymmetric encryption via RSA. We use asymmetric so we can have producers that won't
+    # have ability (when private key not added) to decrypt messages.
+    module Encryption
+      class << self
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:encryption, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          Encryption::Contracts::Config.new.validate!(config.to_h)
+
+          # Don't inject extra components if encryption is not active
+          return unless config.encryption.active
+
+          # This parser is encryption aware
+          config.internal.messages.parser = Messages::Parser.new
+
+          # Encryption for WaterDrop
+          config.producer.middleware.append(Messages::Middleware.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -23,6 +23,11 @@ module Karafka
         processing/jobs/consume_non_blocking
         processing/strategies/base
         routing/features/base
+        encryption
+        encryption/cipher
+        encryption/setup/config
+        encryption/contracts/config
+        encryption/messages/parser
       ].freeze
 
       # Zeitwerk pro loader
@@ -44,14 +49,30 @@ module Karafka
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
-        def setup(config)
+        def pre_setup(config)
+          features.each { |feature| feature.pre_setup(config) }
+
           reconfigure(config)
 
           load_topic_features
         end
 
+        # Runs post setup features configuration operations
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        def post_setup(config)
+          features.each { |feature| feature.post_setup(config) }
+        end
+
         private
 
+        # @return [Array<Module>] extra non-routing related pro features
+        def features
+          [
+            Encryption
+          ]
+        end
+
         # Sets proper config options to use pro components
         # @param config [::Karafka::Core::Configurable::Node] root config node
         def reconfigure(config)

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -42,7 +42,7 @@ module Karafka
                 # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
                 coordinator.pause_tracker.reset
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               else
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -51,7 +51,7 @@ module Karafka
                 # If processing failed, we need to pause
                 # For long running job this will overwrite the default never-ending pause and will
                 # cause the processing to keep going after the error backoff
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end
@@ -61,6 +61,11 @@ module Karafka
             coordinator.on_revoked do
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -47,7 +47,7 @@ module Karafka
 
                 mark_as_consumed(last_group_message)
               else
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -46,6 +46,7 @@ module Karafka
             # This can happen primarily when an LRJ job gets to the internal worker queue and
             # this partition is revoked prior processing.
             unless revoked?
+              Karafka.monitor.instrument('consumer.consume', caller: self)
               Karafka.monitor.instrument('consumer.consumed', caller: self) do
                 consume
               end
@@ -79,7 +80,7 @@ module Karafka
 
                 mark_as_consumed(last_group_message)
               else
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end
@@ -91,6 +92,11 @@ module Karafka
 
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -36,7 +36,7 @@ module Karafka
 
                 mark_as_consumed(messages.last)
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               # If we've reached number of retries that we could, we need to skip the first message
               # that was not marked as consumed, pause and continue, while also moving this message
               # to the dead topic

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -41,7 +41,7 @@ module Karafka
 
                 resume
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               else
                 coordinator.pause_tracker.reset
 

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -38,7 +38,7 @@ module Karafka
 
                 resume
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               else
                 coordinator.pause_tracker.reset
 

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -35,7 +35,7 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               # If we've reached number of retries that we could, we need to skip the first message
               # that was not marked as consumed, pause and continue, while also moving this message
               # to the dead topic.

data/lib/karafka/pro/processing/strategies/lrj.rb

@@ -57,7 +57,7 @@ module Karafka
                 # If processing failed, we need to pause
                 # For long running job this will overwrite the default never-ending pause and will
                 # cause the processing to keep going after the error backoff
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end
@@ -70,6 +70,11 @@ module Karafka
               # a failure. Double non-blocking resume could cause problems in coordination.
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/lrj_mom.rb

@@ -50,7 +50,7 @@ module Karafka
 
                 resume
               else
-                pause(coordinator.seek_offset, false)
+                retry_after_pause
               end
             end
           end
@@ -60,6 +60,11 @@ module Karafka
             coordinator.on_revoked do
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/mom.rb

@@ -32,7 +32,7 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
               else
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -21,9 +21,9 @@ module Karafka
             configure do |config|
               config.error_messages = YAML.safe_load(
                 File.read(
-                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('pro_topic')
+              ).fetch('en').fetch('validations').fetch('topic')
             end
 
             # Make sure that we don't use DLQ with VP

data/lib/karafka/pro/routing/features/long_running_job/contract.rb

@@ -21,9 +21,9 @@ module Karafka
             configure do |config|
               config.error_messages = YAML.safe_load(
                 File.read(
-                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('pro_topic')
+              ).fetch('en').fetch('validations').fetch('topic')
             end
 
             nested(:long_running_job) do

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -21,9 +21,9 @@ module Karafka
             configure do |config|
               config.error_messages = YAML.safe_load(
                 File.read(
-                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('pro_topic')
+              ).fetch('en').fetch('validations').fetch('topic')
             end
 
             nested(:virtual_partitions) do

data/lib/karafka/process.rb

@@ -53,12 +53,14 @@ module Karafka
     # trap context s some things may not work there as expected, that is why we spawn a separate
     # thread to handle the signals process
     def trap_signal(signal)
-      trap(signal) do
+      previous_handler = ::Signal.trap(signal) do
         Thread.new do
           notice_signal(signal)
 
           (@callbacks[signal] || []).each(&:call)
         end
+
+        previous_handler.call if previous_handler.respond_to?(:call)
       end
     end
 

data/lib/karafka/processing/executor.rb

@@ -31,7 +31,7 @@ module Karafka
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
       def initialize(group_id, client, topic)
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @group_id = group_id
         @client = client
         @topic = topic

data/lib/karafka/processing/jobs_queue.rb

@@ -125,13 +125,13 @@ module Karafka
         @semaphores[group_id].pop while wait?(group_id)
       end
 
-      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `busy` - number of jobs that are currently being processed (active work)
       # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
       #
       # @return [Hash] hash with basic usage statistics of this queue.
       def statistics
         {
-          processing: size - @queue.size,
+          busy: size - @queue.size,
           enqueued: @queue.size
         }.freeze
       end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -26,7 +26,7 @@ module Karafka
             # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
             coordinator.pause_tracker.reset
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           else
             coordinator.pause_tracker.reset
             skippable_message = find_skippable_message

data/lib/karafka/processing/strategies/base.rb

@@ -36,6 +36,11 @@ module Karafka
         def handle_revoked
           raise NotImplementedError, 'Implement in a subclass'
         end
+
+        # Shutdown handling
+        def handle_shutdown
+          raise NotImplementedError, 'Implement in a subclass'
+        end
       end
     end
   end

data/lib/karafka/processing/strategies/default.rb

@@ -25,6 +25,7 @@ module Karafka
 
         # Run the user consumption code
         def handle_consume
+          Karafka.monitor.instrument('consumer.consume', caller: self)
           Karafka.monitor.instrument('consumer.consumed', caller: self) do
             consume
           end
@@ -59,7 +60,7 @@ module Karafka
 
             mark_as_consumed(messages.last)
           else
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           end
         end
 
@@ -70,6 +71,19 @@ module Karafka
           resume
 
           coordinator.revoke
+
+          Karafka.monitor.instrument('consumer.revoke', caller: self)
+          Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            revoked
+          end
+        end
+
+        # Runs the shutdown code
+        def handle_shutdown
+          Karafka.monitor.instrument('consumer.shutting_down', caller: self)
+          Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+            shutdown
+          end
         end
       end
     end

data/lib/karafka/processing/strategies/dlq.rb

@@ -26,7 +26,7 @@ module Karafka
 
             mark_as_consumed(messages.last)
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           # If we've reached number of retries that we could, we need to skip the first message
           # that was not marked as consumed, pause and continue, while also moving this message
           # to the dead topic

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -21,7 +21,7 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           # If we've reached number of retries that we could, we need to skip the first message
           # that was not marked as consumed, pause and continue, while also moving this message
           # to the dead topic

data/lib/karafka/processing/strategies/mom.rb

@@ -20,7 +20,7 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
           else
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           end
         end
       end

data/lib/karafka/processing/worker.rb

@@ -25,7 +25,7 @@ module Karafka
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
       end
 
@@ -76,6 +76,8 @@ module Karafka
         Karafka.monitor.instrument(
           'error.occurred',
           caller: self,
+          job: job,
+          jobs_queue: @jobs_queue,
           error: e,
           type: 'worker.process.error'
         )

data/lib/karafka/railtie.rb

@@ -79,6 +79,9 @@ if rails
         ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
           # Reload code each time there is a change in the code
           next unless Rails.application.reloaders.any?(&:updated?)
+          # If consumer persistence is enabled, no reason to reload because we will still keep
+          # old consumer instances in memory.
+          next if Karafka::App.config.consumer_persistence
 
           Rails.application.reloader.reload!
         end

data/lib/karafka/routing/builder.rb

@@ -80,7 +80,7 @@ module Karafka
       # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
       #   a random uuid will be used
       # @param block [Proc] further topics definitions
-      def subscription_group(subscription_group_name = SecureRandom.uuid, &block)
+      def subscription_group(subscription_group_name = SubscriptionGroup.id, &block)
         consumer_group('app') do
           target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
         end

data/lib/karafka/routing/consumer_group.rb

@@ -26,7 +26,7 @@ module Karafka
         @topics = Topics.new([])
         # Initialize the subscription group so there's always a value for it, since even if not
         # defined directly, a subscription group will be created
-        @current_subscription_group_id = SecureRandom.uuid
+        @current_subscription_group_id = SubscriptionGroup.id
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -55,7 +55,7 @@ module Karafka
       # topic definition
       # @param name [String, Symbol] name of the current subscription group
       # @param block [Proc] block that may include topics definitions
-      def subscription_group=(name = SecureRandom.uuid, &block)
+      def subscription_group=(name = SubscriptionGroup.id, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
         @current_subscription_group_id = name
@@ -64,7 +64,7 @@ module Karafka
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        @current_subscription_group_id = SecureRandom.uuid
+        @current_subscription_group_id = SubscriptionGroup.id
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on

data/lib/karafka/routing/consumer_mapper.rb

@@ -12,16 +12,6 @@ module Karafka
     #       raw_consumer_group_name
     #     end
     #   end
-    #
-    # @example Mapper for replacing "_" with "." in topic names
-    #   class MyMapper
-    #     def call(raw_consumer_group_name)
-    #       [
-    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
-    #         raw_consumer_group_name
-    #       ].join('_').gsub('_', '.')
-    #     end
-    #   end
     class ConsumerMapper
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name

data/lib/karafka/routing/features/active_job/contract.rb

@@ -10,7 +10,7 @@ module Karafka
           configure do |config|
             config.error_messages = YAML.safe_load(
               File.read(
-                File.join(Karafka.gem_root, 'config', 'errors.yml')
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               )
             ).fetch('en').fetch('validations').fetch('topic')
           end

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -9,7 +9,7 @@ module Karafka
           configure do |config|
             config.error_messages = YAML.safe_load(
               File.read(
-                File.join(Karafka.gem_root, 'config', 'errors.yml')
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               )
             ).fetch('en').fetch('validations').fetch('topic')
           end

data/lib/karafka/routing/features/manual_offset_management/contract.rb

@@ -9,7 +9,7 @@ module Karafka
           configure do |config|
             config.error_messages = YAML.safe_load(
               File.read(
-                File.join(Karafka.gem_root, 'config', 'errors.yml')
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               )
             ).fetch('en').fetch('validations').fetch('topic')
           end

data/lib/karafka/routing/router.rb

@@ -14,16 +14,26 @@ module Karafka
       # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
       #   any route defined by user using routes.draw
       def find(topic_id)
+        find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
+      end
+
+      # Finds first reference of a given topic based on provided lookup attribute
+      # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
+      # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
+      def find_by(lookup)
         App.consumer_groups.each do |consumer_group|
           consumer_group.topics.each do |topic|
-            return topic if topic.id == topic_id
+            return topic if lookup.all? do |attribute, value|
+              topic.public_send(attribute) == value
+            end
           end
         end
 
-        raise(Errors::NonMatchingRouteError, topic_id)
+        nil
       end
 
       module_function :find
+      module_function :find_by
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -8,7 +8,23 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :name, :topics, :kafka
+      attr_reader :id, :name, :topics, :kafka, :consumer_group
+
+      # Numeric for counting groups
+      GROUP_COUNT = Concurrent::AtomicFixnum.new
+
+      private_constant :GROUP_COUNT
+
+      class << self
+        # Generates new subscription group id that will be used in case of anonymous subscription
+        #   groups
+        # @return [String] hex(6) compatible reproducible id
+        def id
+          ::Digest::MD5.hexdigest(
+            GROUP_COUNT.increment.to_s
+          )[0..11]
+        end
+      end
 
       # @param position [Integer] position of this subscription group in all the subscriptions
       #   groups array. We need to have this value for sake of static group memberships, where
@@ -17,6 +33,7 @@ module Karafka
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
         @name = topics.first.subscription_group
+        @consumer_group = topics.first.consumer_group
         @id = "#{@name}_#{position}"
         @position = position
         @topics = topics

data/lib/karafka/routing/topic.rb

@@ -27,6 +27,7 @@ module Karafka
         @name = name.to_s
         @consumer_group = consumer_group
         @attributes = {}
+        @active = true
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
         #   have same topic name across multiple consumer groups
@@ -66,6 +67,12 @@ module Karafka
         end
       end
 
+      # Allows to disable topic by invoking this method and setting it to `false`.
+      # @param active [Boolean] should this topic be consumed or not
+      def active(active)
+        @active = active
+      end
+
       # @return [Class] consumer class that we should use
       # @note This is just an alias to the `#consumer` method. We however want to use it internally
       #   instead of referencing the `#consumer`. We use this to indicate that this method returns
@@ -77,6 +84,9 @@ module Karafka
 
       # @return [Boolean] should this topic be in use
       def active?
+        # Never active if disabled via routing
+        return false unless @active
+
         topics = Karafka::App.config.internal.routing.active.topics
 
         # When empty it means no topics were specified, hence all should be used
@@ -93,6 +103,7 @@ module Karafka
         Hash[map].merge!(
           id: id,
           name: name,
+          active: active?,
           consumer: consumer,
           consumer_group_id: consumer_group.id,
           subscription_group: subscription_group