karafka 2.0.40 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a4dd6f2d6a1354ac085a5f7ff7de13ca8589d9e83f7fffe2f83c2e4501895a6
-  data.tar.gz: 55426141e6974b329f8e207e20d3acd3935ca025a58b3f1fc287be3c363dee8f
+  metadata.gz: b3d0a2f78b4bf7fa8f49527d48d2e877b95597566e07beabf0166a02259a936b
+  data.tar.gz: fc6054ad5f99bfe8a678c337167f93fc612dddfe88494f8891158dbd8610fb7f
 SHA512:
-  metadata.gz: d5bdbee5398cc6230d31d6915aef359071f68536b89bb7661ce3323a61b926966fe045fd15d3b19b648478a82ce631692302da63e2ccadc8c7c3b476649ae10e
-  data.tar.gz: 924af69aee02bb6a726e09d7f277c8a79dc18eccef6dccf51e62090dc956584ac242bb30b90d0edaec2c626d02960b445ed52e2142b34c20b8b59ce1339b0469
+  metadata.gz: 0fb1fa88ef76ce81e145797a1364ac36bea2b94c47e733856cfd5ec9b37d0d9e2e984a3e4ef7fc36d2ac34c448e490cfdea4e10fae886cd80fb289798e55d308
+  data.tar.gz: 68df2bc1edb9acccd45d32428b43fd5dee12b9333c6059c801aa4ac03b3b89c01e93e8ea4ebbb8021618447c1122c8c5df64afc86747d6f0deec0cf992237e82

data/.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
         run: \curl -sSL https://api.coditsu.io/run/ci | bash
 
   specs:
-    timeout-minutes: 45
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
@@ -102,7 +102,7 @@ jobs:
         run: bin/rspecs
 
   integrations:
-    timeout-minutes: 30
+    timeout-minutes: 45
     runs-on: ubuntu-latest
     needs: diffend
     strategy:

data/CHANGELOG.md

@@ -1,5 +1,34 @@
 # Karafka framework changelog
 
+## 2.1.0 (2023-05-22)
+- **[Feature]** Provide ability to use CurrentAttributes with ActiveJob's Karafka adapter.
+- **[Feature]** Introduce collective Virtual Partitions offset management.
+- **[Feature]** Use virtual offsets to filter out messages that would be re-processed upon retries.
+- [Improvement] No longer break processing on failing parallel virtual partitions in ActiveJob because it is compensated by virtual marking.
+- [Improvement] Always use Virtual offset management for Pro ActiveJobs.
+- [Improvement] Do not attempt to mark offsets on already revoked partitions.
+- [Improvement] Make sure, that VP components are not injected into non VP strategies.
+- [Improvement] Improve complex strategies inheritance flow.
+- [Improvement] Optimize offset management for DLQ + MoM feature combinations.
+- [Change] Removed `Karafka::Pro::BaseConsumer` in favor of `Karafka::BaseConsumer`. (#1345)
+- [Fix] Fix for `max_messages` and `max_wait_time` not having reference in errors.yml (#1443)
+
+### Upgrade notes
+
+1. Upgrade to Karafka `2.0.41` prior to upgrading to `2.1.0`.
+2. Replace `Karafka::Pro::BaseConsumer` references to `Karafka::BaseConsumer`.
+3. Replace `Karafka::Instrumentation::Vendors::Datadog:Listener` with `Karafka::Instrumentation::Vendors::Datadog::MetricsListener`.
+
+## 2.0.41 (2023-14-19)
+- **[Feature]** Provide `Karafka::Pro::Iterator` for anonymous topic/partitions iterations and messages lookups (#1389 and #1427).
+- [Improvement] Optimize topic lookup for `read_topic` admin method usage.
+- [Improvement] Report via `LoggerListener` information about the partition on which a given job has started and finished.
+- [Improvement] Slightly normalize the `LoggerListener` format. Always report partition related operations as followed: `TOPIC_NAME/PARTITION`.
+- [Improvement] Do not retry recovery from `unknown_topic_or_part` when Karafka is shutting down as there is no point and no risk of any data losses.
+- [Improvement] Report `client.software.name` and `client.software.version` according to `librdkafka` recommendation.
+- [Improvement] Report ten longest integration specs after the suite execution.
+- [Improvement] Prevent user originating errors related to statistics processing after listener loop crash from potentially crashing the listener loop and hanging Karafka process.
+
 ## 2.0.40 (2023-04-13)
 - [Improvement] Introduce `Karafka::Messages::Messages#empty?` method to handle Idle related cases where shutdown or revocation would be called on an empty messages set. This method allows for checking if there are any messages in the messages batch.
 - [Refactor] Require messages builder to accept partition and do not fetch it from messages.
@@ -50,7 +79,7 @@
 
 ## 2.0.35 (2023-03-13)
 - **[Feature]** Allow for defining topics config via the DSL and its automatic creation via CLI command.
-- **[Feature]** Allow for full topics reset and topics repartitioning via the CLI. 
+- **[Feature]** Allow for full topics reset and topics repartitioning via the CLI.
 
 ## 2.0.34 (2023-03-04)
 - [Improvement] Attach an `embedded` tag to Karafka processes started using the embedded API.

data/Gemfile

@@ -6,10 +6,11 @@ plugin 'diffend'
 
 gemspec
 
-# Karafka gem does not require this but we add it here so we can test the integration with
-# ActiveJob much easier
+# Karafka gem does not require activejob nor karafka-web  to work
+# They are added here because they are part of the integration suite
 group :integrations do
   gem 'activejob'
+  gem 'karafka-web'
 end
 
 group :test do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.40)
+    karafka (2.1.0)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
@@ -22,6 +22,7 @@ GEM
     concurrent-ruby (1.2.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
+    erubi (1.12.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
@@ -36,9 +37,18 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    karafka-web (0.5.1)
+      erubi (~> 1.4)
+      karafka (>= 2.0.40, < 3.0.0)
+      karafka-core (>= 2.0.12, < 3.0.0)
+      roda (~> 3.63)
+      tilt (~> 2.0)
     mini_portile2 (2.8.1)
     minitest (5.18.0)
+    rack (3.0.7)
     rake (13.0.6)
+    roda (3.67.0)
+      rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -59,6 +69,7 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
+    tilt (2.1.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.5.1)
@@ -75,6 +86,7 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
+  karafka-web
   rspec
   simplecov
 

data/bin/integrations

@@ -152,8 +152,14 @@ class Scenario
     end
   end
 
+  # @return [Float] number of seconds that a given spec took to run
+  def time_taken
+    @finished_at - @started_at
+  end
+
   # Close all the files that are open, so they do not pile up
   def close
+    @finished_at = current_time
     @stdin.close
     @stdout.close
     @stderr.close
@@ -262,13 +268,22 @@ while finished_scenarios.size < scenarios.size
   sleep(0.1)
 end
 
+# Report longest scenarios
+puts
+puts "\nLongest scenarios:\n\n"
+
+finished_scenarios.sort_by(&:time_taken).reverse.first(10).each do |long_scenario|
+  puts "[#{'%6.2f' % long_scenario.time_taken}] #{long_scenario.name}"
+end
+
 failed_scenarios = finished_scenarios.reject(&:success?)
 
-# Report once more on the failed jobs
-# This will only list scenarios that failed without printing their stdout here.
 if failed_scenarios.empty?
   puts
 else
+  # Report once more on the failed jobs
+  # This will only list scenarios that failed without printing their stdout here.
+  puts
   puts "\nFailed scenarios:\n\n"
 
   failed_scenarios.each do |scenario|

data/config/locales/errors.yml

@@ -15,6 +15,13 @@ en:
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
       kafka_format: needs to be a filled hash
+      internal.processing.jobs_builder_format: cannot be nil
+      internal.processing.scheduler: cannot be nil
+      internal.processing.coordinator_class: cannot be nil
+      internal.processing.partitioner_class: cannot be nil
+      internal.active_job.dispatcher: cannot be nil
+      internal.active_job.job_options_contract: cannot be nil
+      internal.active_job.consumer_class: cannot be nil
       internal.status_format: needs to be present
       internal.process_format: needs to be present
       internal.routing.builder_format: needs to be present
@@ -31,7 +38,10 @@ en:
       topics_missing: No topics to subscribe to
 
     topic:
+      kafka: needs to be a hash with kafka scope settings details
       missing: needs to be present
+      max_messages_format: 'needs to be an integer bigger than 0'
+      max_wait_time_format: 'needs to be an integer bigger than 0'
       name_format: 'needs to be a string with a Kafka accepted format'
       deserializer_format: needs to be present
       consumer_format: needs to be present

data/config/locales/pro_errors.yml

@@ -4,8 +4,6 @@ en:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
 
-      manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
-
       long_running_job.active_format: needs to be either true or false
 
       dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.

data/lib/karafka/active_job/consumer.rb

@@ -24,21 +24,26 @@ module Karafka
       #
       # @param job_message [Karafka::Messages::Message] message with active job
       def consume_job(job_message)
-        # We technically speaking could set this as deserializer and reference it from the
-        # message instead of using the `#raw_payload`. This is not done on purpose to simplify
-        # the ActiveJob setup here
-        job = ::ActiveSupport::JSON.decode(job_message.raw_payload)
+        with_deserialized_job(job_message) do |job|
+          tags.add(:job_class, job['job_class'])
 
-        tags.add(:job_class, job['job_class'])
+          payload = { caller: self, job: job, message: job_message }
 
-        payload = { caller: self, job: job, message: job_message }
-
-        # We publish both to make it consistent with `consumer.x` events
-        Karafka.monitor.instrument('active_job.consume', payload)
-        Karafka.monitor.instrument('active_job.consumed', payload) do
-          ::ActiveJob::Base.execute(job)
+          # We publish both to make it consistent with `consumer.x` events
+          Karafka.monitor.instrument('active_job.consume', payload)
+          Karafka.monitor.instrument('active_job.consumed', payload) do
+            ::ActiveJob::Base.execute(job)
+          end
         end
       end
+
+      # @param job_message [Karafka::Messages::Message] message with active job
+      def with_deserialized_job(job_message)
+        # We technically speaking could set this as deserializer and reference it from the
+        # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+        # the ActiveJob setup here
+        yield ::ActiveSupport::JSON.decode(job_message.raw_payload)
+      end
     end
   end
 end

data/lib/karafka/active_job/current_attributes/loading.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    module CurrentAttributes
+      # Module expanding the job deserialization to extract current attributes and load them
+      # for the time of the job execution
+      module Loading
+        # @param job_message [Karafka::Messages::Message] message with active job
+        def with_deserialized_job(job_message)
+          super(job_message) do |job|
+            resetable = []
+
+            _cattr_klasses.each do |key, cattr_klass_str|
+              next unless job.key?(key)
+
+              attributes = job.delete(key)
+
+              cattr_klass = cattr_klass_str.constantize
+
+              attributes.each do |name, value|
+                cattr_klass.public_send("#{name}=", value)
+              end
+
+              resetable << cattr_klass
+            end
+
+            yield(job)
+
+            resetable.each(&:reset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/current_attributes/persistence.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    module CurrentAttributes
+      # Module adding the current attributes persistence into the ActiveJob jobs
+      module Persistence
+        # Alters the job serialization to inject the current attributes into the json before we
+        # send it to Kafka
+        #
+        # @param job [ActiveJob::Base] job
+        def serialize_job(job)
+          json = super(job)
+
+          _cattr_klasses.each do |key, cattr_klass_str|
+            next if json.key?(key)
+
+            attrs = cattr_klass_str.constantize.attributes
+
+            json[key] = attrs unless attrs.empty?
+          end
+
+          json
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/current_attributes.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'active_support/current_attributes'
+require_relative 'current_attributes/loading'
+require_relative 'current_attributes/persistence'
+
+# This code is based on Sidekiqs approach to persisting current attributes
+# @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb
+module Karafka
+  module ActiveJob
+    # Module that allows to persist current attributes on Karafka jobs
+    module CurrentAttributes
+      # Allows for persistence of given current attributes via AJ + Karafka
+      #
+      # @param klasses [Array<String, Class>] classes or names of the current attributes classes
+      def persist(*klasses)
+        # Support for providing multiple classes
+        klasses = Array(klasses).flatten
+
+        [Dispatcher, Consumer]
+          .reject { |expandable| expandable.respond_to?(:_cattr_klasses) }
+          .each { |expandable| expandable.class_attribute :_cattr_klasses, default: {} }
+
+        # Do not double inject in case of running persist multiple times
+        Dispatcher.prepend(Persistence) unless Dispatcher.ancestors.include?(Persistence)
+        Consumer.prepend(Loading) unless Consumer.ancestors.include?(Loading)
+
+        klasses.map(&:to_s).each do |stringified_klass|
+          # Prevent registering same klass multiple times
+          next if Dispatcher._cattr_klasses.value?(stringified_klass)
+
+          key = "cattr_#{Dispatcher._cattr_klasses.count}"
+
+          Dispatcher._cattr_klasses[key] = stringified_klass
+          Consumer._cattr_klasses[key] = stringified_klass
+        end
+      end
+
+      module_function :persist
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -18,7 +18,7 @@ module Karafka
         ::Karafka.producer.public_send(
           fetch_option(job, :dispatch_method, DEFAULTS),
           topic: job.queue_name,
-          payload: ::ActiveSupport::JSON.encode(job.serialize)
+          payload: ::ActiveSupport::JSON.encode(serialize_job(job))
         )
       end
 
@@ -34,7 +34,7 @@ module Karafka
 
           dispatches[d_method] << {
             topic: job.queue_name,
-            payload: ::ActiveSupport::JSON.encode(job.serialize)
+            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
           }
         end
 
@@ -58,6 +58,12 @@ module Karafka
           .karafka_options
           .fetch(key, defaults.fetch(key))
       end
+
+      # @param job [ActiveJob::Base] job
+      # @return [Hash] json representation of the job
+      def serialize_job(job)
+        job.serialize
+      end
     end
   end
 end

data/lib/karafka/admin.rb

@@ -96,13 +96,15 @@ module Karafka
           end
         end
 
+        # Use topic from routes if we can match it or create a dummy one
+        # Dummy one is used in case we cannot match the topic with routes. This can happen
+        # when admin API is used to read topics that are not part of the routing
+        topic = ::Karafka::Routing::Router.find_or_initialize_by_name(name)
+
         messages.map! do |message|
           Messages::Builders::Message.call(
             message,
-            # Use topic from routes if we can match it or create a dummy one
-            # Dummy one is used in case we cannot match the topic with routes. This can happen
-            # when admin API is used to read topics that are not part of the routing
-            Routing::Router.find_by(name: name) || Topic.new(name, App.config.deserializer),
+            topic,
             Time.now
           )
         end
@@ -173,6 +175,17 @@ module Karafka
         end
       end
 
+      # Creates consumer instance and yields it. After usage it closes the consumer instance
+      # This API can be used in other pieces of code and allows for low-level consumer usage
+      #
+      # @param settings [Hash] extra settings to customize consumer
+      def with_consumer(settings = {})
+        consumer = config(:consumer, settings).consumer
+        yield(consumer)
+      ensure
+        consumer&.close
+      end
+
       private
 
       # @return [Array<String>] topics names
@@ -195,15 +208,6 @@ module Karafka
         admin&.close
       end
 
-      # Creates consumer instance and yields it. After usage it closes the consumer instance
-      # @param settings [Hash] extra settings to customize consumer
-      def with_consumer(settings = {})
-        consumer = config(:consumer, settings).consumer
-        yield(consumer)
-      ensure
-        consumer&.close
-      end
-
       # There are some cases where rdkafka admin operations finish successfully but without the
       # callback being triggered to materialize the post-promise object. Until this is fixed we
       # can figure out, that operation we wanted to do finished successfully by checking that the

data/lib/karafka/connection/client.rb

@@ -30,7 +30,7 @@ module Karafka
       #
       # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
       #   with all the configuration details needed for us to create a client
-      # @return [Karafka::Connection::Rdk::Consumer]
+      # @return [Karafka::Connection::Client]
       def initialize(subscription_group)
         @id = SecureRandom.hex(6)
         # Name is set when we build consumer
@@ -369,6 +369,8 @@ module Karafka
       rescue ::Rdkafka::RdkafkaError => e
         early_report = false
 
+        retryable = time_poll.attempts <= MAX_POLL_RETRIES && time_poll.retryable?
+
         # There are retryable issues on which we want to report fast as they are source of
         # problems and can mean some bigger system instabilities
         # Those are mainly network issues and exceeding the max poll interval
@@ -389,9 +391,10 @@ module Karafka
           return nil if @subscription_group.kafka[:'allow.auto.create.topics']
 
           early_report = true
-        end
 
-        retryable = time_poll.attempts <= MAX_POLL_RETRIES && time_poll.retryable?
+          # No sense in retrying when no topic/partition and we're no longer running
+          retryable = false unless Karafka::App.running?
+        end
 
         if early_report || !retryable
           Karafka.monitor.instrument(

data/lib/karafka/errors.rb

@@ -46,5 +46,8 @@ module Karafka
 
     # This should never happen. Please open an issue if it does.
     StrategyNotFoundError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    InvalidRealOffsetUsage = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -32,6 +32,18 @@ module Karafka
             consumer_group_id: @consumer_group_id,
             statistics: @statistics_decorator.call(statistics)
           )
+        # We need to catch and handle any potential errors coming from the instrumentation pipeline
+        # as otherwise, in case of statistics which run in the main librdkafka thread, any crash
+        # will hang the whole process.
+        rescue StandardError => e
+          ::Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            type: 'statistics.emitted.error',
+            error: e
+          )
         end
       end
     end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -48,7 +48,8 @@ module Karafka
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
-        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
+        partition = job.executor.partition
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic}/#{partition} started"
       end
 
       # Prints info about the fact that a given job has finished
@@ -60,7 +61,11 @@ module Karafka
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
-        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
+        partition = job.executor.partition
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{job.id}] #{job_type} job for #{consumer}
+          on #{topic}/#{partition} finished in #{time}ms
+        MSG
       end
 
       # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
@@ -73,7 +78,7 @@ module Karafka
         client = event[:caller]
 
         info <<~MSG.tr("\n", ' ').strip!
-          [#{client.id}] Pausing partition #{partition} of topic #{topic} on offset #{offset}
+          [#{client.id}] Pausing on topic #{topic}/#{partition} on offset #{offset}
         MSG
       end
 
@@ -86,7 +91,7 @@ module Karafka
         client = event[:caller]
 
         info <<~MSG.tr("\n", ' ').strip!
-          [#{client.id}] Resuming partition #{partition} of topic #{topic}
+          [#{client.id}] Resuming on topic #{topic}/#{partition}
         MSG
       end
 
@@ -102,7 +107,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
-          on partition #{partition} of topic #{topic} from offset #{offset}
+          on topic #{topic}/#{partition} from offset #{offset}
         MSG
       end
 
@@ -261,6 +266,12 @@ module Karafka
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details
+        # Those can occur when emitted statistics are consumed by the end user and the processing
+        # of statistics fails. The statistics are emitted from librdkafka main loop thread and
+        # any errors there crash the whole thread
+        when 'statistics.emitted.error'
+          error "statistics.emitted processing failed due to an error: #{error}"
+          error details
         # Those will only occur when retries in the client fail and when they did not stop after
         # back-offs
         when 'connection.client.poll.error'