karafka 2.4.8 → 2.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7e8e51a5c0c4ded0d074965cee2090e64ca77e43443f4f36ab03cc3a21ddfd6
-  data.tar.gz: c6e912c518d301f55974a9e4deb491ebe4c3e073e6748fe62ce1003eaea7bed7
+  metadata.gz: 69bfb8dac259cdb89eadfa86b8a13b7e901fbf82f7562e0869c7bd88f6111f30
+  data.tar.gz: 68959cc13a83db2611a41556ce99db5fed6f98272626dc12f64e4c18415bc633
 SHA512:
-  metadata.gz: 8769a192c7ebb852250afd96611e115807172bf320d4e9d513bffbcc66f5570ca637aee1b8f3b68e46350b37054935ea905893139bc264a02949f975b39f2041
-  data.tar.gz: 13dc8e118850aace7127bae5fe667b6f73118d220b61b1d2b590cc18e71243b3c1467d43df9ae134cd7b26c0274a100e34d8a07db0c6091bc02fd9c95ffb24f0
+  metadata.gz: 54d7bb2ad1d65df9b2e92f28a8ae2255dac95c37fcbc3b1bde554a7d6be0c48c79ee164f6d7b8ebd57d9b09a18ff3f955cc9a193e5ca0207403d503224fc0f61
+  data.tar.gz: 94b08e703378d7b074d32f220dbb24a02a0e3039f948780bb7c434be721df53a52b03ca349d3e26fd8330ff6d7049d389fe27965b8120ef9d4047b1acc3984d1

data/.github/workflows/ci.yml

@@ -77,7 +77,6 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
-          - '3.0'
         include:
           - ruby: '3.3'
             coverage: 'true'

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka Framework Changelog
 
+## 2.4.9 (2024-08-23)
+- **[Feature]** Provide Kafka based Recurring (Cron) Tasks.
+- [Enhancement] Wrap worker work with Rails Reloader/Executor (fusion2004)
+- [Enhancement] Allow for partial topic level kafka scope settings reconfiguration via `inherit` flag.
+- [Enhancement] Validate `eof` kafka scope flag when `eofed` in routing enabled.
+- [Enhancement] Provide `mark_after_dispatch` setting for granular DLQ marking control.
+- [Enhancement] Provide `Karafka::Admin.rename_consumer_group`.
+
 ## 2.4.8 (2024-08-09)
 - **[Feature]** Introduce ability to react to `#eof` either from `#consume` or from `#eofed` when EOF without new messages.
 - [Enhancement] Provide `Consumer#eofed?` to indicate reaching EOF.

data/Gemfile

@@ -6,20 +6,23 @@ plugin 'diffend'
 
 gemspec
 
-# Karafka gem does not require activejob nor karafka-web to work
+# Karafka gem does not require activejob, karafka-web or fugit to work
 # They are added here because they are part of the integration suite
 # Since some of those are only needed for some specs, they should never be required automatically
+group :integrations, :test do
+  gem 'fugit', require: false
+  gem 'rspec', require: false
+end
+
 group :integrations do
   gem 'activejob', require: false
-  gem 'karafka-testing', '>= 2.4.0', require: false
-  gem 'karafka-web', '>= 0.10.0.beta1', require: false
-  gem 'rspec', require: false
+  gem 'karafka-testing', '>= 2.4.6', require: false
+  gem 'karafka-web', '>= 0.10.0.rc2', require: false
 end
 
 group :test do
   gem 'byebug'
   gem 'factory_bot'
   gem 'ostruct'
-  gem 'rspec'
   gem 'simplecov'
 end

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.8)
+    karafka (2.4.9)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -11,31 +11,37 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.1.3.4)
-      activesupport (= 7.1.3.4)
+    activejob (7.2.1)
+      activesupport (= 7.2.1)
       globalid (>= 0.3.6)
-    activesupport (7.1.3.4)
+    activesupport (7.2.1)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.1)
     drb (2.2.1)
     erubi (1.13.0)
+    et-orbi (1.2.11)
+      tzinfo
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     ffi (1.17.0)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
     i18n (1.14.5)
@@ -49,19 +55,20 @@ GEM
     karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
-    karafka-web (0.10.0.rc1)
+    karafka-web (0.10.0)
       erubi (~> 1.4)
       karafka (>= 2.4.7, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
+    logger (1.6.0)
     mini_portile2 (2.8.7)
-    minitest (5.24.1)
-    mutex_m (0.2.0)
+    minitest (5.25.1)
     ostruct (0.6.0)
+    raabro (1.4.0)
     rack (3.1.7)
     rake (13.2.1)
-    roda (3.82.0)
+    roda (3.83.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -76,6 +83,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
+    securerandom (0.3.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -99,9 +107,10 @@ DEPENDENCIES
   activejob
   byebug
   factory_bot
+  fugit
   karafka!
-  karafka-testing (>= 2.4.0)
-  karafka-web (>= 0.10.0.beta1)
+  karafka-testing (>= 2.4.6)
+  karafka-web (>= 0.10.0.rc2)
   ostruct
   rspec
   simplecov

data/bin/integrations

@@ -240,6 +240,11 @@ ARGV.each do |filter|
   end
 end
 
+# Remove Rails 7.2 specs from Ruby 3.0 because it requires 3.1
+specs.delete_if do |spec|
+  RUBY_VERSION < '3.1' && spec.include?('rails72')
+end
+
 raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
 
 # Randomize order

data/certs/cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTI0MDgyMzEwMTkyMFoXDTQ5MDgxNzEwMTkyMFowPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKjLhLjQqUlNayxkXnO+
+PsmCDs/KFIzhrsYMfLZRZNaWmzV3ujljMOdDjd4snM2X06C41iVdQPWjpe3j8vVe
+ZXEWR/twSbOP6Eeg8WVH2wCOo0x5i7yhVn4UBLH4JpfEMCbemVcWQ9ry9OMg4WpH
+Uu4dRwxFV7hzCz3p0QfNLRI4miAxnGWcnlD98IJRjBAksTuR1Llj0vbOrDGsL9ZT
+JeXP2gdRLd8SqzAFJEWrbeTBCBU7gfSh3oMg5SVDLjaqf7Kz5wC/8bDZydzanOxB
+T6CDXPsCnllmvTNx2ei2T5rGYJOzJeNTmJLLK6hJWUlAvaQSvCwZRvFJ0tVGLEoS
+flqSr6uGyyl1eMUsNmsH4BqPEYcAV6P2PKTv2vUR8AP0raDvZ3xL1TKvfRb8xRpo
+vPopCGlY5XBWEc6QERHfVLTIVsjnls2/Ujj4h8/TSfqqYnaHKefIMLbuD/tquMjD
+iWQsW2qStBV0T+U7FijKxVfrfqZP7GxQmDAc9o1iiyAa3QIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU3O4dTXmvE7YpAkszGzR9DdL9
+sbEwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAVKTfoLXn7mqdSxIR
+eqxcR6Huudg1jes81s1+X0uiRTR3hxxKZ3Y82cPsee9zYWyBrN8TA4KA0WILTru7
+Ygxvzha0SRPsSiaKLmgOJ+61ebI4+bOORzIJLpD6GxCxu1r7MI4+0r1u1xe0EWi8
+agkVo1k4Vi8cKMLm6Gl9b3wG9zQBw6fcgKwmpjKiNnOLP+OytzUANrIUJjoq6oal
+TC+f/Uc0TLaRqUaW/bejxzDWWHoM3SU6aoLPuerglzp9zZVzihXwx3jPLUVKDFpF
+Rl2lcBDxlpYGueGo0/oNzGJAAy6js8jhtHC9+19PD53vk7wHtFTZ/0ugDQYnwQ+x
+oml2fAAuVWpTBCgOVFe6XCQpMKopzoxQ1PjKztW2KYxgJdIBX87SnL3aWuBQmhRd
+i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
+ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
+-----END CERTIFICATE-----

data/config/locales/errors.yml

@@ -33,6 +33,8 @@ en:
       internal.processing.partitioner_class_format: cannot be nil
       internal.processing.strategy_selector_format: cannot be nil
       internal.processing.expansions_selector_format: cannot be nil
+      internal.processing.executor_class_format: cannot be nil
+      internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
 
       internal.active_job.dispatcher_format: cannot be nil
       internal.active_job.job_options_contract_format: cannot be nil
@@ -113,10 +115,12 @@ en:
       dead_letter_queue.transactional_format: needs to be either true or false
       dead_letter_queue.dispatch_method_format: 'needs to be either #produce_sync or #produce_async'
       dead_letter_queue.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
+      dead_letter_queue.mark_after_dispatch_format: 'needs to be true, false or nil'
 
       active_format: needs to be either true or false
 
       eofed.active_format: needs to be either true or false
+      eofed.kafka_enable: 'cannot be enabled without enable.partition.eof set to true'
 
       declaratives.partitions_format: needs to be more or equal to 1
       declaratives.active_format: needs to be true

data/config/locales/pro_errors.yml

@@ -63,6 +63,8 @@ en:
       swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
       swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
 
+      recurring_tasks.active_format: 'needs to be boolean'
+
       direct_assignments.active_missing: needs to be present
       direct_assignments.active_format: 'needs to be boolean'
       direct_assignments.partitions_missing: 'needs to be present'
@@ -99,5 +101,20 @@ en:
       patterns.ttl_format: needs to be an integer bigger than 0
       patterns.ttl_missing: needs to be present
 
+      recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
+      recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.schedules_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.logs_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
+      recurring_tasks.deserializer_format: 'needs to be configured'
+      recurring_tasks.logging_format: needs to be a boolean
+
     routing:
       swarm_nodes_not_used: 'At least one of the nodes has no assignments'
+
+    recurring_tasks:
+      id_format: 'can include only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_)'
+      cron_format: must be a non-empty string
+      enabled_format: needs to be a boolean
+      changed_format: needs to be a boolean
+      previous_time_format: needs to be a numerical or time

data/karafka.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
-  spec.cert_chain    = %w[certs/cert_chain.pem]
+  spec.cert_chain    = %w[certs/cert.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]

data/lib/karafka/admin.rb

@@ -274,6 +274,48 @@ module Karafka
         end
       end
 
+      # Takes consumer group and its topics and migrates all the offsets to a new named group
+      #
+      # @param previous_name [String] old consumer group name
+      # @param new_name [String] new consumer group name
+      # @param topics [Array<String>] topics for which we want to migrate offsets during rename
+      # @param delete_previous [Boolean] should we delete previous consumer group after rename.
+      #   Defaults to true.
+      #
+      # @note This method should **not** be executed on a running consumer group as it creates a
+      #   "fake" consumer and uses it to move offsets.
+      #
+      # @note After migration unless `delete_previous` is set to `false`, old group will be
+      #   removed.
+      #
+      # @note If new consumer group exists, old offsets will be added to it.
+      def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
+        remap = Hash.new { |h, k| h[k] = {} }
+
+        old_lags = read_lags_with_offsets({ previous_name => topics })
+
+        return if old_lags.empty?
+
+        read_lags_with_offsets({ previous_name => topics })
+          .fetch(previous_name)
+          .each do |topic, partitions|
+            partitions.each do |partition_id, details|
+              offset = details[:offset]
+
+              # No offset on this partition
+              next if offset.negative?
+
+              remap[topic][partition_id] = offset
+            end
+          end
+
+        seek_consumer_group(new_name, remap)
+
+        return unless delete_previous
+
+        delete_consumer_group(previous_name)
+      end
+
       # Removes given consumer group (if exists)
       #
       # @param consumer_group_id [String] consumer group name

data/lib/karafka/contracts/config.rb

@@ -116,6 +116,8 @@ module Karafka
           required(:partitioner_class) { |val| !val.nil? }
           required(:strategy_selector) { |val| !val.nil? }
           required(:expansions_selector) { |val| !val.nil? }
+          required(:executor_class) { |val| !val.nil? }
+          required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
         end
 
         nested(:active_job) do

data/lib/karafka/errors.rb

@@ -82,10 +82,11 @@ module Karafka
     AssignmentLostError = Class.new(BaseError)
 
     # Raised if optional dependencies like karafka-web are required in a version that is not
-    # supported by the current framework version.
+    # supported by the current framework version or when an optional dependency is missing.
     #
     # Because we do not want to require web out of the box and we do not want to lock web with
-    # karafka 1:1, we do such a sanity check
+    # karafka 1:1, we do such a sanity check. This also applies to cases where some external
+    # optional dependencies are needed but not available.
     DependencyConstraintsError = Class.new(BaseError)
 
     # Raised when we were not able to open pidfd for given pid

data/lib/karafka/pro/loader.rb

@@ -75,7 +75,8 @@ module Karafka
         def features
           [
             Encryption,
-            Cleaner
+            Cleaner,
+            RecurringTasks
           ]
         end
 

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -135,7 +135,12 @@ module Karafka
 
               dispatch = lambda do
                 dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                mark_dispatched_to_dlq(skippable_message)
+
+                if mark_after_dispatch?
+                  mark_dispatched_to_dlq(skippable_message)
+                else
+                  coordinator.seek_offset = skippable_message.offset + 1
+                end
               end
 
               if dispatch_in_a_transaction?
@@ -193,6 +198,16 @@ module Karafka
               producer.transactional? && topic.dead_letter_queue.transactional?
             end
 
+            # @return [Boolean] should we mark given message as consumed after dispatch.
+            #  For default non MOM strategies if user did not explicitly tell us not to, we mark
+            #  it. Default is `nil`, which means `true` in this case. If user provided alternative
+            #  value, we go with it.
+            def mark_after_dispatch?
+              return true if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
+
             # Runs the DLQ strategy and based on it it performs certain operations
             #
             # In case of `:skip` and `:dispatch` will run the exact flow provided in a block

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -55,7 +55,11 @@ module Karafka
                     skippable_message, _marked = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      coordinator.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -46,11 +46,27 @@ module Karafka
                     skippable_message, _marked = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      coordinator.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end
             end
+
+            # @return [Boolean] should we mark given message as consumed after dispatch. For
+            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+            #  is `nil`, which means `false` in this case. If user provided alternative value, we
+            #  go with it.
+            #
+            # @note Please note, this is the opposite behavior than in case of AOM strategies.
+            def mark_after_dispatch?
+              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -49,11 +49,27 @@ module Karafka
                     skippable_message, _marked = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      coordinator.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end
             end
+
+            # @return [Boolean] should we mark given message as consumed after dispatch. For
+            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+            #  is `nil`, which means `false` in this case. If user provided alternative value, we
+            #  go with it.
+            #
+            # @note Please note, this is the opposite behavior than in case of AOM strategies.
+            def mark_after_dispatch?
+              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -40,16 +40,32 @@ module Karafka
                     skippable_message, = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    # Save the next offset we want to go with after moving given message to DLQ
-                    # Without this, we would not be able to move forward and we would end up
-                    # in an infinite loop trying to un-pause from the message we've already
-                    # processed. Of course, since it's a MoM a rebalance or kill, will move it back
-                    # as no offsets are being committed
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      # Save the next offset we want to go with after moving given message to DLQ
+                      # Without this, we would not be able to move forward and we would end up
+                      # in an infinite loop trying to un-pause from the message we've already
+                      # processed. Of course, since it's a MoM a rebalance or kill, will move it
+                      # back as no offsets are being committed
+                      coordinator.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end
             end
+
+            # @return [Boolean] should we mark given message as consumed after dispatch. For
+            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+            #  is `nil`, which means `false` in this case. If user provided alternative value, we
+            #  go with it.
+            #
+            # @note Please note, this is the opposite behavior than in case of AOM strategies.
+            def mark_after_dispatch?
+              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
           end
         end
       end

data/lib/karafka/pro/recurring_tasks/consumer.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Consumer responsible for management of the recurring tasks and their execution
+      # There are some assumptions made here that always need to be satisfied:
+      #   - we only run schedules that are of same or newer version
+      #   - we always mark as consumed in such a way, that the first message received after
+      #     assignment (if any) is a state
+      class Consumer < ::Karafka::BaseConsumer
+        # @param args [Array] all arguments accepted by the consumer
+        def initialize(*args)
+          super
+          @executor = Executor.new
+        end
+
+        def consume
+          # There is nothing we can do if we operate on a newer schedule. In such cases we should
+          # just wait and re-raise error hoping someone will notice or that this will be
+          # reassigned to a process with newer schedule
+          raise(Errors::IncompatibleScheduleError) if @executor.incompatible?
+
+          messages.each do |message|
+            payload = message.payload
+            type = payload[:type]
+
+            case type
+            when 'schedule'
+              # If we're replaying data, we need to record the most recent stored state, so we
+              # can use this data to fully initialize the scheduler
+              @executor.update_state(payload) if @executor.replaying?
+
+              # If this is first message we cannot mark it on the previous offset
+              next if message.offset.zero?
+
+              # We always mark as consumed in such a way, that when replaying, we start from a
+              # schedule state message. This makes it easier to recover.
+              mark_as_consumed Karafka::Messages::Seek.new(
+                topic.name,
+                partition,
+                message.offset - 1
+              )
+            when 'command'
+              @executor.apply_command(payload)
+
+              next if @executor.replaying?
+
+              # Execute on each incoming command to have nice latency but only after replaying
+              # During replaying we should not execute because there may be more state changes
+              # that collectively have a different outcome
+              @executor.call
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError, type
+            end
+          end
+
+          eofed if eofed?
+        end
+
+        # Starts the final replay process if we reached eof during replaying
+        def eofed
+          # We only mark as replayed if we were replaying in the first place
+          # If already replayed, nothing to do
+          return unless @executor.replaying?
+
+          @executor.replay
+        end
+
+        # Runs the cron execution if all good.
+        def tick
+          # Do nothing until we fully recover the correct state
+          return if @executor.replaying?
+
+          # If the state is incompatible, we can only raise an error.
+          # We do it here and in the `#consume` so the pause-retry kicks in basically reporting
+          # on this issue once every minute. That way user should not miss this.
+          # We use seek to move so we can achieve a pause of 60 seconds in between consecutive
+          # errors instead of on each tick because it is much more frequent.
+          if @executor.incompatible?
+            if messages.empty?
+              raise Errors::IncompatibleScheduleError
+            else
+              return seek(messages.last.offset - 1)
+            end
+          end
+
+          # If all good and compatible we can execute the recurring tasks
+          @executor.call
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:recurring_tasks) do
+            required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
+            required(:deserializer) { |val| !val.nil? }
+            required(:logging) { |val| [true, false].include?(val) }
+            # Do not allow to run more often than every 5 seconds
+            required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+            required(:group_id) do |val|
+              val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+            end
+
+            nested(:topics) do
+              required(:schedules) do |val|
+                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              end
+
+              required(:logs) do |val|
+                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end