karafka 2.4.8 → 2.4.9

data/lib/karafka/railtie.rb

@@ -65,28 +65,6 @@ if Karafka.rails?
         app.config.autoload_paths += %w[app/consumers]
       end
 
-      initializer 'karafka.configure_rails_code_reloader' do
-        # There are components that won't work with older Rails version, so we check it and
-        # provide a failover
-        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
-
-        next unless Rails.env.development?
-        next unless ENV.key?('KARAFKA_CLI')
-        next unless rails6plus
-
-        # We can have many listeners, but it does not matter in which we will reload the code
-        # as long as all the consumers will be re-created as Rails reload is thread-safe
-        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-          # If consumer persistence is enabled, no reason to reload because we will still keep
-          # old consumer instances in memory.
-          next if Karafka::App.config.consumer_persistence
-          # Reload code each time there is a change in the code
-          next unless Rails.application.reloaders.any?(&:updated?)
-
-          Rails.application.reloader.reload!
-        end
-      end
-
       initializer 'karafka.release_active_record_connections' do
         rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
 
@@ -136,6 +114,17 @@ if Karafka.rails?
           end
         end
       end
+
+      initializer 'karafka.configure_worker_external_executor' do |app|
+        app.config.after_initialize do
+          app_config = Karafka::App.config
+
+          # We need to wrap execution of the core user code with a wrapper in case of Rails, so
+          # the auto-reload works as expected
+          worker_job_call_wrapper = app_config.consumer_persistence ? app.executor : app.reloader
+          app_config.internal.processing.worker_job_call_wrapper = worker_job_call_wrapper
+        end
+      end
     end
   end
 end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -21,6 +21,9 @@ module Karafka
           :dispatch_method,
           # Should we use `#mark_as_consumed` or `#mark_as_consumed!` (in flows that mark)
           :marking_method,
+          # Should we mark as consumed after dispatch or not. True for most cases, except MOM where
+          # it is on user to decide (false by default)
+          :mark_after_dispatch,
           # Initialize with kwargs
           keyword_init: true
         ) do

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -21,6 +21,7 @@ module Karafka
               required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
               required(:transactional) { |val| [true, false].include?(val) }
+              required(:mark_after_dispatch) { |val| [true, false, nil].include?(val) }
 
               required(:dispatch_method) do |val|
                 %i[produce_async produce_sync].include?(val)

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -22,6 +22,9 @@ module Karafka
           #   whether dispatch on dlq should be sync or async (async by default)
           # @param marking_method [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Describes
           #   whether marking on DLQ should be async or sync (async by default)
+          # @param mark_after_dispatch [Boolean, nil] Should we mark after dispatch. `nil` means
+          #   that the default strategy approach to marking will be used. `true` or `false`
+          #   overwrites the default
           # @return [Config] defined config
           def dead_letter_queue(
             max_retries: DEFAULT_MAX_RETRIES,
@@ -29,7 +32,8 @@ module Karafka
             independent: false,
             transactional: true,
             dispatch_method: :produce_async,
-            marking_method: :mark_as_consumed
+            marking_method: :mark_as_consumed,
+            mark_after_dispatch: nil
           )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
@@ -38,7 +42,8 @@ module Karafka
               independent: independent,
               transactional: transactional,
               dispatch_method: dispatch_method,
-              marking_method: marking_method
+              marking_method: marking_method,
+              mark_after_dispatch: mark_after_dispatch
             )
           end
 

data/lib/karafka/routing/features/eofed/contracts/topic.rb

@@ -19,6 +19,18 @@ module Karafka
             nested :eofed do
               required(:active) { |val| [true, false].include?(val) }
             end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              eofed = data[:eofed]
+
+              next unless eofed[:active]
+
+              next if data[:kafka][:'enable.partition.eof']
+
+              [[%i[eofed kafka], :enable]]
+            end
           end
         end
       end

data/lib/karafka/routing/topic.rb

@@ -54,6 +54,20 @@ module Karafka
         RUBY
       end
 
+      # Often users want to have the same basic cluster setup with small setting alterations
+      # This method allows us to do so by setting `inherit` to `true`. Whe inherit is enabled,
+      # settings will be merged with defaults.
+      #
+      # @param settings [Hash] kafka scope settings. If `:inherit` key is provided, it will
+      #   instruct the assignment to merge with root level defaults
+      #
+      # @note It is set to `false` by default to preserve backwards compatibility
+      def kafka=(settings = {})
+        inherit = settings.delete(:inherit)
+
+        @kafka = inherit ? Karafka::App.config.kafka.merge(settings) : settings
+      end
+
       # @return [String] name of subscription that will go to librdkafka
       def subscription_name
         name

data/lib/karafka/setup/config.rb

@@ -285,6 +285,9 @@ module Karafka
           setting :expansions_selector, default: Processing::ExpansionsSelector.new
           # option [Class] executor class
           setting :executor_class, default: Processing::Executor
+          # option worker_job_call_wrapper [Proc, false] callable object that will be used to wrap
+          #   the worker execution of a job or false if no wrapper needed
+          setting :worker_job_call_wrapper, default: false
         end
 
         # Things related to operating on messages

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.8'
+  VERSION = '2.4.9'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.8
+  version: 2.4.9
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -12,30 +12,30 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
   YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
+  MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
   dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-  8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-  sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-  wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-  b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-  ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-  5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-  YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-  kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-  1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-  bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-  3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-  kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-  IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-  fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-  qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+  PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+  ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+  Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+  JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+  T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+  flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+  vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+  iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+  sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+  eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+  Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+  agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+  TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+  Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+  oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+  i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+  ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-08-09 00:00:00.000000000 Z
+date: 2024-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -163,7 +163,7 @@ files:
 - bin/stress_one
 - bin/verify_license_integrity
 - bin/wait_for_kafka
-- certs/cert_chain.pem
+- certs/cert.pem
 - certs/karafka-pro.pem
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
@@ -370,6 +370,20 @@ files:
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
 - lib/karafka/pro/processing/subscription_groups_coordinator.rb
+- lib/karafka/pro/recurring_tasks.rb
+- lib/karafka/pro/recurring_tasks/consumer.rb
+- lib/karafka/pro/recurring_tasks/contracts/config.rb
+- lib/karafka/pro/recurring_tasks/contracts/task.rb
+- lib/karafka/pro/recurring_tasks/deserializer.rb
+- lib/karafka/pro/recurring_tasks/dispatcher.rb
+- lib/karafka/pro/recurring_tasks/errors.rb
+- lib/karafka/pro/recurring_tasks/executor.rb
+- lib/karafka/pro/recurring_tasks/listener.rb
+- lib/karafka/pro/recurring_tasks/matcher.rb
+- lib/karafka/pro/recurring_tasks/schedule.rb
+- lib/karafka/pro/recurring_tasks/serializer.rb
+- lib/karafka/pro/recurring_tasks/setup/config.rb
+- lib/karafka/pro/recurring_tasks/task.rb
 - lib/karafka/pro/routing/features/active_job.rb
 - lib/karafka/pro/routing/features/active_job/builder.rb
 - lib/karafka/pro/routing/features/base.rb
@@ -434,6 +448,12 @@ files:
 - lib/karafka/pro/routing/features/periodic_job/config.rb
 - lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb
 - lib/karafka/pro/routing/features/periodic_job/topic.rb
+- lib/karafka/pro/routing/features/recurring_tasks.rb
+- lib/karafka/pro/routing/features/recurring_tasks/builder.rb
+- lib/karafka/pro/routing/features/recurring_tasks/config.rb
+- lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb
+- lib/karafka/pro/routing/features/recurring_tasks/proxy.rb
+- lib/karafka/pro/routing/features/recurring_tasks/topic.rb
 - lib/karafka/pro/routing/features/swarm.rb
 - lib/karafka/pro/routing/features/swarm/config.rb
 - lib/karafka/pro/routing/features/swarm/contracts/routing.rb

data/certs/cert_chain.pem

@@ -1,26 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
-YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
-dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
------END CERTIFICATE-----