karafka 2.0.31 → 2.0.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4c598dcc6414c2a24f50452d5379c2a0aa26d7d1c7fd8d1b07f2af19a6e8125
-  data.tar.gz: 184a8b1e7bb1f62672f90904ce6c3fd5e2da5f1868bcb2a971a1e7872e8c3252
+  metadata.gz: 511cc39ce92e82706fcd6ac9c54b3416c7908cf5e9773ee15b972219046778ca
+  data.tar.gz: 737059dc4f9de33577f4883590f607da1ee4ee330ff9f5c18bd42dfe261c567d
 SHA512:
-  metadata.gz: a9d382efa4846f4419d86d1fa2742f663e2c255b203612b6810ab5bc44aafc8c9a1c6c299330889c0cc54cb952235ddbb8d45a3c2ed30c7132b8522fc46ef8d1
-  data.tar.gz: 5a5907e0e7b958a6277785ef4254ac5d8cf7f563e371fdf3fe7705719164362858b46efae62dda6c7a1f02646ffd57d05437fde496ebf2b7e36bdcb8ac57865d
+  metadata.gz: d91e5de5bddd7b9ff2e423c986eda7f8e795528dc5ffb7a3694c82a2a096f8b3ee31679ac221bb0da4cd6efc7efa55ab195f081fbd2cf393df7d21549d950cd9
+  data.tar.gz: dbea8026eb3385b6ee14a749b3bc0187f4e91e9e66b60f2e8ac74fcdd409ad238b8e6f7dd5e5d5fbb5f0cc36299795ed63b794f1dd4a2a2b53e26776a7523876

data/CHANGELOG.md

@@ -1,6 +1,47 @@
 # Karafka framework changelog
 
-## 2.0.31 (2022-02-12)
+## 2.0.33 (2023-02-24)
+- **[Feature]** Support `perform_all_later` in ActiveJob adapter for Rails `7.1+`
+- **[Feature]** Introduce ability to assign and re-assign tags in consumer instances. This can be used for extra instrumentation that is context aware.
+- **[Feature]** Introduce ability to assign and reassign tags to the `Karafka::Process`.
+- [Improvement] When using `ActiveJob` adapter, automatically tag jobs with the name of the `ActiveJob` class that is running inside of the `ActiveJob` consumer.
+- [Improvement] Make `::Karafka::Instrumentation::Notifications::EVENTS` list public for anyone wanting to re-bind those into a different notification bus.
+- [Improvement] Set `fetch.message.max.bytes` for `Karafka::Admin` to `5MB` to make sure that all data is fetched correctly for Web UI under heavy load (many consumers).
+- [Improvement] Introduce a `strict_topics_namespacing` config option to enable/disable the strict topics naming validations. This can be useful when working with pre-existing topics which we cannot or do not want to rename.
+- [Fix] Karafka monitor is prematurely cached (#1314)
+
+### Upgrade notes
+
+Since `#tags` were introduced on consumers, the `#tags` method is now part of the consumers API.
+
+This means, that in case you were using a method called `#tags` in your consumers, you will have to rename it:
+
+```ruby
+class EventsConsumer < ApplicationConsumer
+  def consume
+    messages.each do |message|
+      tags << message.payload.tag
+    end
+
+    tags.each { |tags| puts tag }
+  end
+
+  private
+
+  # This will collide with the tagging API
+  # This NEEDS to be renamed not to collide with `#tags` method provided by the consumers API.
+  def tags
+    @tags ||= Set.new
+  end
+end
+```
+
+## 2.0.32 (2023-02-13)
+- [Fix] Many non-existing topic subscriptions propagate poll errors beyond client
+- [Improvement] Ignore `unknown_topic_or_part` errors in dev when `allow.auto.create.topics` is on.
+- [Improvement] Optimize temporary errors handling in polling for a better backoff policy
+
+## 2.0.31 (2023-02-12)
 - [Feature] Allow for adding partitions via `Admin#create_partitions` API.
 - [Fix] Do not ignore admin errors upon invalid configuration (#1254)
 - [Fix] Topic name validation (#1300) - CandyFet
@@ -8,7 +49,7 @@
 - [Maintenance] Require `karafka-core` >= `2.0.11` and switch to shared RSpec locator.
 - [Maintenance] Require `karafka-rdkafka` >= `0.12.1`
 
-## 2.0.30 (2022-01-31)
+## 2.0.30 (2023-01-31)
 - [Improvement] Alias `--consumer-groups` with `--include-consumer-groups`
 - [Improvement] Alias `--subscription-groups` with `--include-subscription-groups`
 - [Improvement] Alias `--topics` with `--include-topics`

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.31)
-      karafka-core (>= 2.0.11, < 3.0.0)
+    karafka (2.0.33)
+      karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -29,7 +29,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.11)
+    karafka-core (2.0.12)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.12.1)
     karafka-rdkafka (0.12.1)
@@ -64,10 +64,9 @@ GEM
     waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.6)
+    zeitwerk (2.6.7)
 
 PLATFORMS
-  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES

data/config/locales/errors.yml

@@ -45,18 +45,23 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       active_format: needs to be either true or false
-      inconsistent_namespacing: needs to be consistent namespacing style
+      inconsistent_namespacing: |
+        needs to be consistent namespacing style
+        disable this validation by setting config.strict_topics_namespacing to false
 
     consumer_group:
       missing: needs to be present
       topics_names_not_unique: all topic names within a single consumer group must be unique
-      topics_namespaced_names_not_unique: all topic names within a single consumer group must be unique considering namespacing styles
       id_format: 'needs to be a string with a Kafka accepted format'
       topics_format: needs to be a non-empty array
+      topics_namespaced_names_not_unique: |
+        all topic names within a single consumer group must be unique considering namespacing styles
+        disable this validation by setting config.strict_topics_namespacing to false
 
     job_options:
       missing: needs to be present
       dispatch_method_format: needs to be either :produce_async or :produce_sync
+      dispatch_many_method_format: needs to be either :produce_many_async or :produce_many_sync
       partitioner_format: 'needs to respond to #call'
       partition_key_type_format: 'needs to be either :key or :partition_key'
 

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.11', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.4.10', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -11,7 +11,13 @@ module ActiveJob
       #
       # @param job [Object] job that should be enqueued
       def enqueue(job)
-        ::Karafka::App.config.internal.active_job.dispatcher.call(job)
+        ::Karafka::App.config.internal.active_job.dispatcher.dispatch(job)
+      end
+
+      # Enqueues multiple jobs in one go
+      # @param jobs [Array<Object>] jobs that we want to enqueue
+      def enqueue_all(jobs)
+        ::Karafka::App.config.internal.active_job.dispatcher.dispatch_many(jobs)
       end
 
       # Raises info, that Karafka backend does not support scheduling jobs

data/lib/karafka/active_job/consumer.rb

@@ -12,12 +12,14 @@ module Karafka
         messages.each do |message|
           break if Karafka::App.stopping?
 
-          ::ActiveJob::Base.execute(
-            # We technically speaking could set this as deserializer and reference it from the
-            # message instead of using the `#raw_payload`. This is not done on purpose to simplify
-            # the ActiveJob setup here
-            ::ActiveSupport::JSON.decode(message.raw_payload)
-          )
+          # We technically speaking could set this as deserializer and reference it from the
+          # message instead of using the `#raw_payload`. This is not done on purpose to simplify
+          # the ActiveJob setup here
+          job = ::ActiveSupport::JSON.decode(message.raw_payload)
+
+          tags.add(:job_class, job['job_class'])
+
+          ::ActiveJob::Base.execute(job)
 
           mark_as_consumed(message)
         end

data/lib/karafka/active_job/dispatcher.rb

@@ -7,13 +7,14 @@ module Karafka
       # Defaults for dispatching
       # The can be updated by using `#karafka_options` on the job
       DEFAULTS = {
-        dispatch_method: :produce_async
+        dispatch_method: :produce_async,
+        dispatch_many_method: :produce_many_async
       }.freeze
 
       private_constant :DEFAULTS
 
       # @param job [ActiveJob::Base] job
-      def call(job)
+      def dispatch(job)
         ::Karafka.producer.public_send(
           fetch_option(job, :dispatch_method, DEFAULTS),
           topic: job.queue_name,
@@ -21,6 +22,30 @@ module Karafka
         )
       end
 
+      # Bulk dispatches multiple jobs using the Rails 7.1+ API
+      # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
+      def dispatch_many(jobs)
+        # Group jobs by their desired dispatch method
+        # It can be configured per job class, so we need to make sure we divide them
+        dispatches = Hash.new { |hash, key| hash[key] = [] }
+
+        jobs.each do |job|
+          d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+
+          dispatches[d_method] << {
+            topic: job.queue_name,
+            payload: ::ActiveSupport::JSON.encode(job.serialize)
+          }
+        end
+
+        dispatches.each do |type, messages|
+          ::Karafka.producer.public_send(
+            type,
+            messages
+          )
+        end
+      end
+
       private
 
       # @param job [ActiveJob::Base] job

data/lib/karafka/active_job/job_options_contract.rb

@@ -15,7 +15,18 @@ module Karafka
         ).fetch('en').fetch('validations').fetch('job_options')
       end
 
-      optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
+      optional(:dispatch_method) do |val|
+        %i[
+          produce_async
+          produce_sync
+        ].include?(val)
+      end
+      optional(:dispatch_many_method) do |val|
+        %i[
+          produce_many_async
+          produce_many_sync
+        ].include?(val)
+      end
     end
   end
 end

data/lib/karafka/admin.rb

@@ -26,7 +26,9 @@ module Karafka
       'group.id': 'karafka_admin',
       # We want to know when there is no more data not to end up with an endless loop
       'enable.partition.eof': true,
-      'statistics.interval.ms': 0
+      'statistics.interval.ms': 0,
+      # Fetch at most 5 MBs when using admin
+      'fetch.message.max.bytes': 5 * 1_048_576
     }.freeze
 
     private_constant :Topic, :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS

data/lib/karafka/base_consumer.rb

@@ -4,6 +4,9 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
+    # Allow for consumer instance tagging for instrumentation
+    include ::Karafka::Core::Taggable
+
     # @return [String] id of the current consumer
     attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed

data/lib/karafka/connection/client.rb

@@ -380,6 +380,14 @@ module Karafka
         when :network_exception # 13
           early_report = true
         when :transport # -195
+          early_report = true
+        # @see
+        # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
+        # This will be raised each time poll detects a non-existing topic. When auto creation is
+        # on, we can safely ignore it
+        when :unknown_topic_or_part # 3
+          return nil if @subscription_group.kafka[:'allow.auto.create.topics']
+
           early_report = true
         end
 
@@ -423,8 +431,7 @@ module Karafka
           Instrumentation::Callbacks::Statistics.new(
             @subscription_group.id,
             @subscription_group.consumer_group_id,
-            @name,
-            ::Karafka::App.config.monitor
+            @name
           )
         )
 
@@ -434,8 +441,7 @@ module Karafka
           Instrumentation::Callbacks::Error.new(
             @subscription_group.id,
             @subscription_group.consumer_group_id,
-            @name,
-            ::Karafka::App.config.monitor
+            @name
           )
         )
 

data/lib/karafka/contracts/consumer_group.rb

@@ -27,6 +27,7 @@ module Karafka
 
       virtual do |data, errors|
         next unless errors.empty?
+        next unless ::Karafka::App.config.strict_topics_namespacing
 
         names = data.fetch(:topics).map { |topic| topic[:name] }
         names_hash = names.each_with_object({}) { |n, h| h[n] = true }

data/lib/karafka/contracts/topic.rb

@@ -51,6 +51,7 @@ module Karafka
 
       virtual do |data, errors|
         next unless errors.empty?
+        next unless ::Karafka::App.config.strict_topics_namespacing
 
         value = data.fetch(:name)
         namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -9,12 +9,10 @@ module Karafka
         # @param subscription_group_id [String] id of the current subscription group instance
         # @param consumer_group_id [String] id of the current consumer group
         # @param client_name [String] rdkafka client name
-        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
-        def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
+        def initialize(subscription_group_id, consumer_group_id, client_name)
           @subscription_group_id = subscription_group_id
           @consumer_group_id = consumer_group_id
           @client_name = client_name
-          @monitor = monitor
         end
 
         # Runs the instrumentation monitor with error
@@ -26,7 +24,7 @@ module Karafka
           # Same as with statistics (mor explanation there)
           return unless @client_name == client_name
 
-          @monitor.instrument(
+          ::Karafka.monitor.instrument(
             'error.occurred',
             caller: self,
             subscription_group_id: @subscription_group_id,

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -10,12 +10,10 @@ module Karafka
         # @param subscription_group_id [String] id of the current subscription group
         # @param consumer_group_id [String] id of the current consumer group
         # @param client_name [String] rdkafka client name
-        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
-        def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
+        def initialize(subscription_group_id, consumer_group_id, client_name)
           @subscription_group_id = subscription_group_id
           @consumer_group_id = consumer_group_id
           @client_name = client_name
-          @monitor = monitor
           @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
         end
 
@@ -28,7 +26,7 @@ module Karafka
           # all the time.
           return unless @client_name == statistics['name']
 
-          @monitor.instrument(
+          ::Karafka.monitor.instrument(
             'statistics.emitted',
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,

data/lib/karafka/instrumentation/notifications.rb

@@ -54,8 +54,6 @@ module Karafka
         error.occurred
       ].freeze
 
-      private_constant :EVENTS
-
       # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
         super

data/lib/karafka/pro/active_job/consumer.rb

@@ -31,9 +31,11 @@ module Karafka
             break if revoked?
             break if Karafka::App.stopping?
 
-            ::ActiveJob::Base.execute(
-              ::ActiveSupport::JSON.decode(message.raw_payload)
-            )
+            job = ::ActiveSupport::JSON.decode(message.raw_payload)
+
+            tags.add(:job_class, job['job_class'])
+
+            ::ActiveJob::Base.execute(job)
 
             # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
             # this could create random markings.

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -23,6 +23,7 @@ module Karafka
         # They can be updated by using `#karafka_options` on the job
         DEFAULTS = {
           dispatch_method: :produce_async,
+          dispatch_many_method: :produce_many_async,
           # We don't create a dummy proc based partitioner as we would have to evaluate it with
           # each job.
           partitioner: nil,
@@ -33,7 +34,7 @@ module Karafka
         private_constant :DEFAULTS
 
         # @param job [ActiveJob::Base] job
-        def call(job)
+        def dispatch(job)
           ::Karafka.producer.public_send(
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
@@ -43,6 +44,28 @@ module Karafka
           )
         end
 
+        # Bulk dispatches multiple jobs using the Rails 7.1+ API
+        # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
+        def dispatch_many(jobs)
+          dispatches = Hash.new { |hash, key| hash[key] = [] }
+
+          jobs.each do |job|
+            d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+
+            dispatches[d_method] << dispatch_details(job).merge!(
+              topic: job.queue_name,
+              payload: ::ActiveSupport::JSON.encode(job.serialize)
+            )
+          end
+
+          dispatches.each do |type, messages|
+            ::Karafka.producer.public_send(
+              type,
+              messages
+            )
+          end
+        end
+
         private
 
         # @param job [ActiveJob::Base] job instance

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -25,9 +25,20 @@ module Karafka
           ).fetch('en').fetch('validations').fetch('job_options')
         end
 
-        optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
         optional(:partitioner) { |val| val.respond_to?(:call) }
         optional(:partition_key_type) { |val| %i[key partition_key].include?(val) }
+        optional(:dispatch_method) do |val|
+          %i[
+            produce_async
+            produce_sync
+          ].include?(val)
+        end
+        optional(:dispatch_many_method) do |val|
+          %i[
+            produce_many_async
+            produce_many_sync
+          ].include?(val)
+        end
       end
     end
   end

data/lib/karafka/process.rb

@@ -4,6 +4,9 @@ module Karafka
   # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
+    # Allow for process tagging for instrumentation
+    extend ::Karafka::Core::Taggable
+
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
       SIGINT

data/lib/karafka/setup/config.rb

@@ -89,6 +89,11 @@ module Karafka
       # option [::WaterDrop::Producer, nil]
       # Unless configured, will be created once Karafka is configured based on user Karafka setup
       setting :producer, default: nil
+      # option [Boolean] when set to true, Karafka will ensure that the routing topic naming
+      # convention is strict
+      # Disabling this may be needed in scenarios where we do not have control over topics names
+      # and/or we work with existing systems where we cannot change topics names.
+      setting :strict_topics_namespacing, default: true
 
       # rdkafka default options
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
@@ -168,6 +173,9 @@ module Karafka
 
           configure_components
 
+          # Refreshes the references that are cached that might have been changed by the config
+          ::Karafka.refresh!
+
           # Runs things that need to be executed after config is defined and all the components
           # are also configured
           Pro::Loader.post_setup(config) if Karafka.pro?

data/lib/karafka/time_trackers/poll.rb

@@ -51,6 +51,9 @@ module Karafka
       # Sleeps for amount of time matching attempt, so we sleep more with each attempt in case of
       #   a retry.
       def backoff
+        # backoff should not be included in the remaining time computation, otherwise it runs
+        # shortly, never back-offing beyond a small number because of the sleep
+        @remaining += backoff_interval
         # Sleep requires seconds not ms
         sleep(backoff_interval / 1_000.0)
       end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.31'
+  VERSION = '2.0.33'
 end

data/lib/karafka.rb

@@ -95,6 +95,19 @@ module Karafka
     def boot_file
       Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
     end
+
+    # We need to be able to overwrite both monitor and logger after the configuration in case they
+    # would be changed because those two (with defaults) can be used prior to the setup and their
+    # state change should be reflected in the updated setup
+    #
+    # This method refreshes the things that might have been altered by the configuration
+    def refresh!
+      config = ::Karafka::App.config
+
+      @logger = config.logger
+      @producer = config.producer
+      @monitor = config.monitor
+    end
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.31
+  version: 2.0.33
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-02-13 00:00:00.000000000 Z
+date: 2023-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.11
+        version: 2.0.12
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.11
+        version: 2.0.12
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0