karafka 2.0.23 → 2.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84d8130c528081b283889f9f1ebe89b8829b800a2c5d1f4ca99f6d6ce9b4c9df
-  data.tar.gz: 3cc30f65586226bcb6d8ed4fdac912a72c1a4a7eb5691862b378629d9de1347f
+  metadata.gz: 25eb837923f78f1bf35402bf24469228d8f5af12090b2d406718a0077ce42598
+  data.tar.gz: 23520ecab430080061046e683c0dcb47316d1216d1ad9843894caefff76e24cc
 SHA512:
-  metadata.gz: 8e1f3fc0a3c73035fdb38093eb0594c999d0a715752f41ed95079b1d8f2e89a5a0fa0abdf054272891e89ffa4dd8d010dd2225540d4480067d8bc0359bf2b7b8
-  data.tar.gz: 7c1f3f958ef52682e46efb11e8a3ddcbe641ba85c62956fefb84ec6eacadbe9d28ee509bdfd61fcd73ea8f52b1ab867de4909990503024cc5c603788931264ca
+  metadata.gz: 3e84c4c4dedd7a160ebcbd6cff53e0dfcf601a31eb19469d1a86eabb3f33507f3b1871c8f4489b743651118ed47e52925f12dba4fc67e462bd8be04d71dfa3bb
+  data.tar.gz: 8b9754f5566c5ef213c4803cfba2a4263553a5112e23fd12e42a963fd493f8f20c1409b12303628519ba7ef5bcf07aa09fabc6dd4ffc2a2114a6bad8f94cbc72

data/.github/workflows/ci.yml

@@ -46,6 +46,7 @@ jobs:
         run: \curl -sSL https://api.coditsu.io/run/ci | bash
 
   specs:
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
@@ -84,6 +85,7 @@ jobs:
         run: bin/rspecs
 
   integrations:
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: diffend
     strategy:

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Karafka framework changelog
 
+## 2.0.24 (2022-12-19)
+- **[Feature]** Provide out of the box encryption support for Pro.
+- [Improvement] Add instrumentation upon `#pause`.
+- [Improvement] Add instrumentation upon retries.
+- [Improvement] Assign `#id` to consumers similar to other entities for ease of debugging.
+- [Improvement] Add retries and pausing to the default `LoggerListener`.
+- [Improvement] Introduce a new final `terminated` state that will kick in prior to exit but after all the instrumentation and other things are done.
+- [Improvement] Ensure that state transitions are thread-safe and ensure state transitions can occur in one direction.
+- [Improvement] Optimize status methods proxying to `Karafka::App`.
+- [Improvement] Allow for easier state usage by introducing explicit `#to_s` for reporting.
+- [Improvement] Change auto-generated id from `SecureRandom#uuid` to `SecureRandom#hex(6)`
+- [Improvement] Emit statistic every 5 seconds by default.
+- [Improvement] Introduce general messages parser that can be swapped when needed.
+- [Fix] Do not trigger code reloading when `consumer_persistence` is enabled.
+- [Fix] Shutdown producer after all the consumer components are down and the status is stopped. This will ensure, that any instrumentation related Kafka messaging can still operate.
+
+### Upgrade notes
+
+If you want to disable `librdkafka` statistics because you do not use them at all, update the `kafka` `statistics.interval.ms` setting and set it to `0`:
+
+```ruby
+class KarafkaApp < Karafka::App
+  setup do |config|
+    # Other settings...
+    config.kafka = {
+      'statistics.interval.ms': 0
+    }
+  end
+end
+```
+
 ## 2.0.23 (2022-12-07)
 - [Maintenance] Align with `waterdrop` and `karafka-core`
 - [Improvement] Provide `Admin#read_topic` API to get topic data without subscribing.
@@ -425,7 +456,7 @@ There are several things in the plan already for 2.1 and beyond, including a web
 - Small integration specs refactoring + specs for pausing scenarios
 
 ## 2.0.0-alpha6 (2022-04-17)
-- Fix a bug, where upon missing boot file and Rails, railtie would fail with a generic exception (#818) 
+- Fix a bug, where upon missing boot file and Rails, railtie would fail with a generic exception (#818)
 - Fix an issue with parallel pristine specs colliding with each other during `bundle install` (#820)
 - Replace `consumer.consume` with `consumer.consumed` event to match the behaviour
 - Make sure, that offset committing happens before the `consumer.consumed` event is propagated

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.23)
-      karafka-core (>= 2.0.6, < 3.0.0)
+    karafka (2.0.24)
+      karafka-core (>= 2.0.7, < 3.0.0)
       thor (>= 0.20)
-      waterdrop (>= 2.4.3, < 3.0.0)
+      waterdrop (>= 2.4.7, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -29,7 +29,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.6)
+    karafka-core (2.0.7)
       concurrent-ruby (>= 1.1)
       rdkafka (>= 0.12)
     mini_portile2 (2.8.0)
@@ -48,7 +48,7 @@ GEM
     rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
+    rspec-mocks (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
@@ -61,12 +61,13 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.3)
-      karafka-core (>= 2.0.6, < 3.0.0)
+    waterdrop (2.4.7)
+      karafka-core (>= 2.0.7, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.6)
 
 PLATFORMS
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES

data/README.md

@@ -72,10 +72,10 @@ Karafka.producer.produce_sync(topic: 'example', payload: { 'ping' => 'pong' }.to
 ```bash
 bundle exec karafka server
 
-[7616dc24-505a-417f-b87b-6bf8fc2d98c5] Polled 1 message in 1000ms
-[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example started
+[86d47f0b92f7] Polled 1 message in 1000ms
+[3732873c8a74] Consume job for ExampleConsumer on example started
 {"ping"=>"pong"}
-[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example finished in 0ms
+[3732873c8a74] Consume job for ExampleConsumer on example finished in 0ms
 ```
 
 ## Want to Upgrade? LGPL is not for you? Want to help?

data/config/{errors.yml → locales/errors.yml}

@@ -63,7 +63,7 @@ en:
 
     pro_topic:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
-      virtual_partitions.max_partitions_format: needs to be equl or more than 1
+      virtual_partitions.max_partitions_format: needs to be equal or more than 1
       manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
       long_running_job.active_format: needs to be either true or false
       dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions

data/config/locales/pro_errors.yml

@@ -0,0 +1,18 @@
+en:
+  validations:
+    topic:
+      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      virtual_partitions.max_partitions_format: needs to be equal or more than 1
+      manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+      long_running_job.active_format: needs to be either true or false
+      dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+
+    config:
+      encryption.active_format: 'needs to be either true or false'
+      encryption.public_key_invalid: 'is not a valid public RSA key'
+      encryption.public_key_needs_to_be_public: 'is a private RSA key not a public one'
+      encryption.private_keys_format: 'needs to be a hash of version and private key value'
+      encryption.private_keys_need_to_be_private: 'all keys need to be private'
+      encryption.version_format: must be a non-empty string
+      encryption.public_key_format: 'is not a valid public RSA key'
+      encryption.private_keys_invalid: 'contains an invalid private RSA key string'

data/docker-compose.yml

@@ -3,8 +3,10 @@ services:
   zookeeper:
     container_name: karafka_20_zookeeper
     image: wurstmeister/zookeeper
+    restart: on-failure
     ports:
       - '2181:2181'
+
   kafka:
     container_name: karafka_20_kafka
     image: wurstmeister/kafka
@@ -22,3 +24,4 @@ services:
          benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    restart: on-failure

data/karafka.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.6', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.7', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.3', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.7', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.7.0'

data/lib/karafka/active_job/job_options_contract.rb

@@ -10,7 +10,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('job_options')
       end

data/lib/karafka/admin.rb

@@ -116,8 +116,7 @@ module Karafka
       # Creates admin instance and yields it. After usage it closes the admin instance
       def with_admin
         admin = config(:producer).admin
-        result = yield(admin)
-        result
+        yield(admin)
       ensure
         admin&.close
       end
@@ -125,8 +124,7 @@ module Karafka
       # Creates consumer instance and yields it. After usage it closes the consumer instance
       def with_consumer
         consumer = config(:consumer).consumer
-        result = yield(consumer)
-        result
+        yield(consumer)
       ensure
         consumer&.close
       end

data/lib/karafka/app.rb

@@ -35,10 +35,21 @@ module Karafka
       # Just a nicer name for the consumer groups
       alias routes consumer_groups
 
-      Status.instance_methods(false).each do |delegated|
-        define_method(delegated) do
-          App.config.internal.status.send(delegated)
-        end
+      # Allow for easier status management via `Karafka::App` by aliasing status methods here
+      Status::STATES.each do |state, transition|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{state}
+            App.config.internal.status.#{state}
+          end
+
+          def #{state}?
+            App.config.internal.status.#{state}?
+          end
+
+          def #{transition}
+            App.config.internal.status.#{transition}
+          end
+        RUBY
       end
 
       # Methods that should be delegated to Karafka module

data/lib/karafka/base_consumer.rb

@@ -4,6 +4,8 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
+    # @return [String] id of the current consumer
+    attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
     attr_accessor :topic
     # @return [Karafka::Messages::Messages] current messages batch
@@ -15,6 +17,11 @@ module Karafka
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
 
+    # Creates new consumer and assigns it an id
+    def initialize
+      @id = SecureRandom.hex(6)
+    end
+
     # Can be used to run preparation code prior to the job being enqueued
     #
     # @private
@@ -204,6 +211,17 @@ module Karafka
 
       # Indicate, that user took a manual action of pausing
       coordinator.manual_pause if manual_pause
+
+      Karafka.monitor.instrument(
+        'consumer.consuming.pause',
+        caller: self,
+        manual: manual_pause,
+        topic: messages.metadata.topic,
+        partition: messages.metadata.partition,
+        offset: offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
     end
 
     # Resumes processing of the current topic partition
@@ -232,5 +250,23 @@ module Karafka
     def revoked?
       coordinator.revoked?
     end
+
+    # Pauses the processing from the last offset to retry on given message
+    # @private
+    def retry_after_pause
+      pause(coordinator.seek_offset, nil, false)
+
+      # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
+      # set by `#pause`
+      Karafka.monitor.instrument(
+        'consumer.consuming.retry',
+        caller: self,
+        topic: messages.metadata.topic,
+        partition: messages.metadata.partition,
+        offset: coordinator.seek_offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
   end
 end

data/lib/karafka/connection/listener.rb

@@ -21,7 +21,7 @@ module Karafka
       def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
         proc_config = ::Karafka::App.config.internal.processing
 
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue

data/lib/karafka/contracts/config.rb

@@ -12,7 +12,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('config')
       end

data/lib/karafka/contracts/consumer_group.rb

@@ -7,7 +7,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('consumer_group')
       end

data/lib/karafka/contracts/server_cli_options.rb

@@ -7,7 +7,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('server_cli_options')
       end

data/lib/karafka/contracts/topic.rb

@@ -7,7 +7,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('topic')
       end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -63,6 +63,38 @@ module Karafka
         info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
+      # Prints info about a pause occurrence. Irrelevant if user or system initiated.
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_consumer_consuming_pause(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        offset = event[:offset]
+        consumer = event[:caller]
+        timeout = event[:timeout]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Pausing partition #{partition} of topic #{topic}
+          on offset #{offset} for #{timeout} ms.
+        MSG
+      end
+
+      # Prints info about retry of processing after an error
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_consumer_consuming_retry(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        offset = event[:offset]
+        consumer = event[:caller]
+        timeout = event[:timeout]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
+          on partition #{partition} of topic #{topic} from offset #{offset}
+        MSG
+      end
+
       # Logs info about system signals that Karafka received and prints backtrace for threads in
       # case of ttin
       #

data/lib/karafka/instrumentation/notifications.rb

@@ -22,8 +22,11 @@ module Karafka
         app.quieting
         app.stopping
         app.stopped
+        app.terminated
 
         consumer.consumed
+        consumer.consuming.pause
+        consumer.consuming.retry
         consumer.revoked
         consumer.shutdown
 

data/lib/karafka/messages/message.rb

@@ -9,7 +9,19 @@ module Karafka
     class Message
       extend Forwardable
 
-      attr_reader :raw_payload, :metadata
+      class << self
+        # @return [Object] general parser
+        # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
+        #   via the config chain.
+        def parser
+          @parser ||= App.config.internal.messages.parser
+        end
+      end
+
+      attr_reader :metadata
+      # raw payload needs to be mutable as we want to have option to change it in the parser
+      # prior to the final deserialization
+      attr_accessor :raw_payload
 
       def_delegators :metadata, *Metadata.members
 
@@ -42,7 +54,7 @@ module Karafka
 
       # @return [Object] deserialized data
       def deserialize
-        metadata.deserializer.call(self)
+        self.class.parser.call(self)
       end
     end
   end

data/lib/karafka/messages/parser.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Default message parser. The only thing it does, is calling the deserializer
+    class Parser
+      # @param message [::Karafka::Messages::Message]
+      # @return [Object] deserialized payload
+      def call(message)
+        message.metadata.deserializer.call(message)
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -20,7 +20,7 @@ module Karafka
         configure do |config|
           config.error_messages = YAML.safe_load(
             File.read(
-              File.join(Karafka.gem_root, 'config', 'errors.yml')
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
             )
           ).fetch('en').fetch('validations').fetch('job_options')
         end

data/lib/karafka/pro/encryption/cipher.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Cipher for encrypting and decrypting data
+      class Cipher
+        def initialize
+          @private_pems = {}
+        end
+
+        # Encrypts given string content with the public key
+        # @param content [String]
+        # @return [String]
+        def encrypt(content)
+          public_pem.public_encrypt(content)
+        end
+
+        # Decrypts provided content using `version` key
+        # @param version [String] encryption version
+        # @param content [String] encrypted content
+        # @return [String] decrypted content
+        def decrypt(version, content)
+          private_pem(version).private_decrypt(content)
+        end
+
+        private
+
+        # @return [::OpenSSL::PKey::RSA] rsa public key
+        def public_pem
+          @public_pem ||= ::OpenSSL::PKey::RSA.new(::Karafka::App.config.encryption.public_key)
+        end
+
+        # @param version [String] version for which we want to get the rsa key
+        # @return [::OpenSSL::PKey::RSA] rsa private key
+        def private_pem(version)
+          return @private_pems[version] if @private_pems.key?(version)
+
+          key_string = ::Karafka::App.config.encryption.private_keys[version]
+          key_string || raise(Errors::PrivateKeyNotFound, version)
+
+          @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:encryption) do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:version) { |val| val.is_a?(String) && !val.empty? }
+            required(:public_key) { |val| val.is_a?(String) }
+
+            required(:private_keys) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(String) } &&
+                val.values.all? { |key| key.is_a?(String) }
+            end
+          end
+
+          # Public key validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            key = OpenSSL::PKey::RSA.new(data.fetch(:encryption).fetch(:public_key))
+
+            next unless key.private?
+
+            [[%i[encryption public_key], :needs_to_be_public]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption public_key], :invalid]]
+          end
+
+          # Private keys validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            private_keys = data.fetch(:encryption).fetch(:private_keys)
+
+            # Keys may be empty for production only envs
+            next if private_keys.empty?
+
+            keys = private_keys.each_value.map do |key|
+              OpenSSL::PKey::RSA.new(key)
+            end
+
+            next if keys.all?(&:private?)
+
+            [[%i[encryption private_keys], :need_to_be_private]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption private_keys], :invalid]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/errors.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related errors
+      module Errors
+        # Raised when we have encountered encryption key with version we do not have
+        PrivateKeyNotFound = Class.new(::Karafka::Errors::BaseError)
+      end
+    end
+  end
+end