karafka 2.0.23 → 2.0.26

data/lib/karafka/connection/listener.rb

@@ -21,7 +21,7 @@ module Karafka
       def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
         proc_config = ::Karafka::App.config.internal.processing
 
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
@@ -85,7 +85,7 @@ module Karafka
       #   propagate this far.
       def fetch_loop
         # Run the main loop as long as we are not stopping or moving into quiet mode
-        until Karafka::App.stopping? || Karafka::App.quieting?
+        until Karafka::App.stopping? || Karafka::App.quieting? || Karafka::App.quiet?
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,
@@ -156,8 +156,9 @@ module Karafka
         # within this consumer group
         @consumer_group_coordinator.finish_work(id)
 
-        # Wait if we're in the quiet mode
-        wait_pinging(wait_until: -> { !Karafka::App.quieting? })
+        # Wait if we're in the process of finishing started work or finished all the work and
+        # just sitting and being quiet
+        wait_pinging(wait_until: -> { !(Karafka::App.quieting? || Karafka::App.quiet?) })
 
         # We need to wait until all the work in the whole consumer group (local to the process)
         # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`

data/lib/karafka/connection/listeners_batch.rb

@@ -6,14 +6,20 @@ module Karafka
     class ListenersBatch
       include Enumerable
 
+      attr_reader :coordinators
+
       # @param jobs_queue [JobsQueue]
       # @return [ListenersBatch]
       def initialize(jobs_queue)
+        @coordinators = []
+
         @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
           consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
             subscription_groups.size
           )
 
+          @coordinators << consumer_group_coordinator
+
           subscription_groups.map do |subscription_group|
             Connection::Listener.new(
               consumer_group_coordinator,

data/lib/karafka/contracts/config.rb

@@ -12,7 +12,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('config')
       end

data/lib/karafka/contracts/consumer_group.rb

@@ -7,7 +7,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('consumer_group')
       end

data/lib/karafka/contracts/server_cli_options.rb

@@ -7,7 +7,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
@@ -80,6 +80,7 @@ module Karafka
       # Makes sure we have anything to subscribe to when we start the server
       virtual do |_, errors|
         next unless errors.empty?
+
         next unless Karafka::App.subscription_groups.empty?
 
         [[%i[topics], :topics_missing]]

data/lib/karafka/contracts/topic.rb

@@ -7,12 +7,11 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('topic')
       end
 
-      required(:consumer) { |val| !val.nil? }
       required(:deserializer) { |val| !val.nil? }
       required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
@@ -20,8 +19,20 @@ module Karafka
       required(:initial_offset) { |val| %w[earliest latest].include?(val) }
       required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
       required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:active) { |val| [true, false].include?(val) }
       required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
 
+      # Consumer needs to be present only if topic is active
+      # We allow not to define consumer for non-active because they may be only used via admin
+      # api or other ways and not consumed with consumer
+      virtual do |data, errors|
+        next unless errors.empty?
+        next if data.fetch(:consumer)
+        next unless data.fetch(:active)
+
+        [[%w[consumer], :missing]]
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
 

data/lib/karafka/instrumentation/logger_listener.rb

@@ -63,6 +63,49 @@ module Karafka
         info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
+      # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_client_pause(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        offset = event[:offset]
+        client = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{client.id}] Pausing partition #{partition} of topic #{topic} on offset #{offset}
+        MSG
+      end
+
+      # Prints information about resuming of processing of a given topic partition
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_client_resume(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        client = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{client.id}] Resuming partition #{partition} of topic #{topic}
+        MSG
+      end
+
+      # Prints info about retry of processing after an error
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_consumer_consuming_retry(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        offset = event[:offset]
+        consumer = event[:caller]
+        timeout = event[:timeout]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
+          on partition #{partition} of topic #{topic} from offset #{offset}
+        MSG
+      end
+
       # Logs info about system signals that Karafka received and prints backtrace for threads in
       # case of ttin
       #
@@ -96,12 +139,17 @@ module Karafka
 
         return if Karafka.pro?
 
-        info 'See LICENSE and the LGPL-3.0 for licensing details.'
+        info 'See LICENSE and the LGPL-3.0 for licensing details'
       end
 
       # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_quieting(_event)
-        info 'Switching to quiet mode. New messages will not be processed.'
+        info 'Switching to quiet mode. New messages will not be processed'
+      end
+
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quiet(_event)
+        info 'Reached quiet mode. No messages will be processed anymore'
       end
 
       # Logs info that we're going to stop the Karafka server.

data/lib/karafka/instrumentation/notifications.rb

@@ -20,27 +20,37 @@ module Karafka
         app.initialized
         app.running
         app.quieting
+        app.quiet
         app.stopping
         app.stopped
+        app.terminated
 
-        consumer.consumed
-        consumer.revoked
-        consumer.shutdown
-
-        process.notice_signal
+        client.pause
+        client.resume
 
         connection.listener.before_fetch_loop
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
+        consumer.consume
+        consumer.consumed
+        consumer.consuming.pause
+        consumer.consuming.retry
+        consumer.revoke
+        consumer.revoked
+        consumer.shutting_down
+        consumer.shutdown
+
         dead_letter_queue.dispatched
 
+        process.notice_signal
+
+        statistics.emitted
+
         worker.process
         worker.processed
         worker.completed
 
-        statistics.emitted
-
         error.occurred
       ].freeze
 

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -4,22 +4,13 @@ module Karafka
   module Instrumentation
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
-      # Updates proc title to an initializing one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_initializing(_event)
-        setproctitle('initializing')
-      end
-
-      # Updates proc title to a running one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_running(_event)
-        setproctitle('running')
-      end
-
-      # Updates proc title to a stopping one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopping(_event)
-        setproctitle('stopping')
+      Status::STATES.each_key do |state|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          # Updates proc title to an appropriate state
+          def on_app_#{state}(_event)
+            setproctitle('#{state}')
+          end
+        RUBY
       end
 
       private

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -148,7 +148,7 @@ module Karafka
             jq_stats = event[:jobs_queue].statistics
 
             gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
-            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
             histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
           end
 
@@ -158,7 +158,7 @@ module Karafka
           def on_worker_processed(event)
             jq_stats = event[:jobs_queue].statistics
 
-            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
           end
 
           private

data/lib/karafka/messages/message.rb

@@ -9,7 +9,19 @@ module Karafka
     class Message
       extend Forwardable
 
-      attr_reader :raw_payload, :metadata
+      class << self
+        # @return [Object] general parser
+        # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
+        #   via the config chain.
+        def parser
+          @parser ||= App.config.internal.messages.parser
+        end
+      end
+
+      attr_reader :metadata
+      # raw payload needs to be mutable as we want to have option to change it in the parser
+      # prior to the final deserialization
+      attr_accessor :raw_payload
 
       def_delegators :metadata, *Metadata.members
 
@@ -42,7 +54,7 @@ module Karafka
 
       # @return [Object] deserialized data
       def deserialize
-        metadata.deserializer.call(self)
+        self.class.parser.call(self)
       end
     end
   end

data/lib/karafka/messages/parser.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Default message parser. The only thing it does, is calling the deserializer
+    class Parser
+      # @param message [::Karafka::Messages::Message]
+      # @return [Object] deserialized payload
+      def call(message)
+        message.metadata.deserializer.call(message)
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -20,7 +20,7 @@ module Karafka
         configure do |config|
           config.error_messages = YAML.safe_load(
             File.read(
-              File.join(Karafka.gem_root, 'config', 'errors.yml')
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
             )
           ).fetch('en').fetch('validations').fetch('job_options')
         end

data/lib/karafka/pro/encryption/cipher.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Cipher for encrypting and decrypting data
+      class Cipher
+        def initialize
+          @private_pems = {}
+        end
+
+        # Encrypts given string content with the public key
+        # @param content [String]
+        # @return [String]
+        def encrypt(content)
+          public_pem.public_encrypt(content)
+        end
+
+        # Decrypts provided content using `version` key
+        # @param version [String] encryption version
+        # @param content [String] encrypted content
+        # @return [String] decrypted content
+        def decrypt(version, content)
+          private_pem(version).private_decrypt(content)
+        end
+
+        private
+
+        # @return [::OpenSSL::PKey::RSA] rsa public key
+        def public_pem
+          @public_pem ||= ::OpenSSL::PKey::RSA.new(::Karafka::App.config.encryption.public_key)
+        end
+
+        # @param version [String] version for which we want to get the rsa key
+        # @return [::OpenSSL::PKey::RSA] rsa private key
+        def private_pem(version)
+          return @private_pems[version] if @private_pems.key?(version)
+
+          key_string = ::Karafka::App.config.encryption.private_keys[version]
+          key_string || raise(Errors::PrivateKeyNotFound, version)
+
+          @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:encryption) do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:version) { |val| val.is_a?(String) && !val.empty? }
+            required(:public_key) { |val| val.is_a?(String) }
+
+            required(:private_keys) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(String) } &&
+                val.values.all? { |key| key.is_a?(String) }
+            end
+          end
+
+          # Public key validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            key = OpenSSL::PKey::RSA.new(data.fetch(:encryption).fetch(:public_key))
+
+            next unless key.private?
+
+            [[%i[encryption public_key], :needs_to_be_public]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption public_key], :invalid]]
+          end
+
+          # Private keys validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            private_keys = data.fetch(:encryption).fetch(:private_keys)
+
+            # Keys may be empty for production only envs
+            next if private_keys.empty?
+
+            keys = private_keys.each_value.map do |key|
+              OpenSSL::PKey::RSA.new(key)
+            end
+
+            next if keys.all?(&:private?)
+
+            [[%i[encryption private_keys], :need_to_be_private]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption private_keys], :invalid]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/errors.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related errors
+      module Errors
+        # Raised when we have encountered encryption key with version we do not have
+        PrivateKeyNotFound = Class.new(::Karafka::Errors::BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related messages components
+      module Messages
+        # Middleware for WaterDrop. It automatically encrypts messages payload.
+        # It is injected only if encryption is enabled.
+        class Middleware
+          # @param message [Hash] WaterDrop message hash
+          # @return [Hash] hash with encrypted payload and encryption version indicator
+          def call(message)
+            message[:headers] ||= {}
+            message[:headers]['encryption'] = version
+            message[:payload] = cipher.encrypt(message[:payload])
+            message
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [String] encryption version
+          def version
+            @version ||= ::Karafka::App.config.encryption.version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      module Messages
+        # Pro parser that takes into consideration encryption usage
+        # @note There may be a case where someone decides not to encrypt data and we start getting
+        #   unencrypted payloads. That is why we always rely on message headers for encryption
+        #   indication.
+        class Parser < ::Karafka::Messages::Parser
+          # @param message [::Karafka::Messages::Message]
+          # @return [Object] deserialized payload
+          def call(message)
+            if active? && message.headers.key?('encryption')
+              # Decrypt raw payload so it can be handled by the default parser logic
+              message.raw_payload = cipher.decrypt(
+                message.headers['encryption'],
+                message.raw_payload
+              )
+            end
+
+            super(message)
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [Boolean] is encryption active
+          def active?
+            return @active unless @active.nil?
+
+            @active = ::Karafka::App.config.encryption.active
+
+            @active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/setup/config.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Setup and config related encryption components
+      module Setup
+        # Config for encryption
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          # Should this feature be in use
+          setting(:active, default: false)
+
+          # Supporting versions allows us to be able to rotate private and public keys in case
+          # we would need this. We can increase the version, rotate and Karafka when decrypting
+          # will figure out proper private key based on the version
+          setting(:version, default: '1')
+
+          # We always support one public key for producing messages
+          # Public key needs to be always present even if we do not plan to produce messages from
+          # a Karafka process. This is because of the web-ui and potentially other cases like this
+          setting(:public_key, default: '')
+
+          # Private keys in pem format, where the key is the version and value is the key.
+          # This allows us to support key rotation
+          setting(:private_keys, default: {})
+
+          # Cipher used to encrypt and decrypt data
+          setting(:cipher, default: Encryption::Cipher.new)
+
+          configure
+        end
+      end
+    end
+  end
+end