karafka 2.0.40 → 2.1.0

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -29,16 +29,18 @@ module Karafka
             # We reset the pause to indicate we will now consider it as "ok".
             coordinator.pause_tracker.reset
 
-            skippable_message, marked = find_skippable_message
+            skippable_message, = find_skippable_message
 
             dispatch_to_dlq(skippable_message)
 
-            # Backoff and move forward
-            if marked
-              pause(coordinator.seek_offset, nil, false)
-            else
-              pause(skippable_message.offset + 1, nil, false)
-            end
+            # Save the next offset we want to go with after moving given message to DLQ
+            # Without this, we would not be able to move forward and we would end up
+            # in an infinite loop trying to un-pause from the message we've already processed
+            # Of course, since it's a MoM a rebalance or kill, will move it back as no
+            # offsets are being committed
+            coordinator.seek_offset = skippable_message.offset + 1
+
+            pause(coordinator.seek_offset, nil, false)
           end
         end
       end

data/lib/karafka/routing/router.rb

@@ -32,8 +32,23 @@ module Karafka
         nil
       end
 
+      # Finds the topic by name (in any consumer group) and if not present, will built a new
+      # representation of the topic with the defaults and default deserializer.
+      #
+      # This is used in places where we may operate on topics that are not part of the routing
+      # but we want to do something on them (display data, iterate over, etc)
+      # @param name [String] name of the topic we are looking for
+      # @return [Karafka::Routing::Topic]
+      #
+      # @note Please note, that in case of a new topic, it will have a newly built consumer group
+      #   as well, that is not part of the routing.
+      def find_or_initialize_by_name(name)
+        find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
+      end
+
       module_function :find
       module_function :find_by
+      module_function :find_or_initialize_by_name
     end
   end
 end

data/lib/karafka/setup/config.rb

@@ -18,7 +18,13 @@ module Karafka
       KAFKA_DEFAULTS = {
         # We emit the statistics by default, so all the instrumentation and web-ui work out of
         # the box, without requiring users to take any extra actions aside from enabling.
-        'statistics.interval.ms': 5_000
+        'statistics.interval.ms': 5_000,
+        'client.software.name': 'karafka',
+        'client.software.version': [
+          "v#{Karafka::VERSION}",
+          "rdkafka-ruby-v#{Rdkafka::VERSION}",
+          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
+        ].join('-')
       }.freeze
 
       # Contains settings that should not be used in production but make life easier in dev

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.40'
+  VERSION = '2.1.0'
 end

data/lib/karafka.rb

@@ -114,6 +114,11 @@ end
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
+# Do not load CurrentAttributes components as they will be loaded if needed
+# @note We have to exclude both the .rb file as well as the whole directory so users can require
+# current attributes only when needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes'))
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes.rb'))
 # Do not load Railtie. It will load if after everything is ready, so we don't have to load any
 # Karafka components when we require this railtie. Railtie needs to be loaded last.
 loader.ignore(Karafka.gem_root.join('lib/karafka/railtie'))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.40
+  version: 2.1.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-04-13 00:00:00.000000000 Z
+date: 2023-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -156,6 +156,9 @@ files:
 - lib/active_job/queue_adapters/karafka_adapter.rb
 - lib/karafka.rb
 - lib/karafka/active_job/consumer.rb
+- lib/karafka/active_job/current_attributes.rb
+- lib/karafka/active_job/current_attributes/loading.rb
+- lib/karafka/active_job/current_attributes/persistence.rb
 - lib/karafka/active_job/dispatcher.rb
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
@@ -197,9 +200,9 @@ files:
 - lib/karafka/instrumentation/notifications.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
-- lib/karafka/instrumentation/vendors/datadog/listener.rb
 - lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb
+- lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -216,7 +219,6 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
-- lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/encryption.rb
 - lib/karafka/pro/encryption/cipher.rb
 - lib/karafka/pro/encryption/contracts/config.rb
@@ -224,6 +226,7 @@ files:
 - lib/karafka/pro/encryption/messages/middleware.rb
 - lib/karafka/pro/encryption/messages/parser.rb
 - lib/karafka/pro/encryption/setup/config.rb
+- lib/karafka/pro/iterator.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/collapser.rb
@@ -232,6 +235,7 @@ files:
 - lib/karafka/pro/processing/filters/delayer.rb
 - lib/karafka/pro/processing/filters/expirer.rb
 - lib/karafka/pro/processing/filters/throttler.rb
+- lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
@@ -261,26 +265,35 @@ files:
 - lib/karafka/pro/processing/strategies/dlq/ftr.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/lrj.rb
 - lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb
+- lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/mom.rb
+- lib/karafka/pro/processing/strategies/dlq/mom_vp.rb
 - lib/karafka/pro/processing/strategies/dlq/vp.rb
 - lib/karafka/pro/processing/strategies/ftr/default.rb
 - lib/karafka/pro/processing/strategies/ftr/vp.rb
 - lib/karafka/pro/processing/strategies/lrj/default.rb
 - lib/karafka/pro/processing/strategies/lrj/ftr.rb
 - lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb
+- lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb
 - lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb
 - lib/karafka/pro/processing/strategies/lrj/mom.rb
+- lib/karafka/pro/processing/strategies/lrj/mom_vp.rb
 - lib/karafka/pro/processing/strategies/lrj/vp.rb
 - lib/karafka/pro/processing/strategies/mom/default.rb
 - lib/karafka/pro/processing/strategies/mom/ftr.rb
+- lib/karafka/pro/processing/strategies/mom/ftr_vp.rb
+- lib/karafka/pro/processing/strategies/mom/vp.rb
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
+- lib/karafka/pro/processing/virtual_offset_manager.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
 - lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'metrics_listener'
-
-module Karafka
-  module Instrumentation
-    # Namespace for vendor specific instrumentation
-    module Vendors
-      # Datadog specific instrumentation
-      module Datadog
-        # Alias to keep backwards compatibility
-        Listener = MetricsListener
-      end
-    end
-  end
-end