karafka 2.4.9 → 2.4.10

data/lib/karafka/pro/scheduled_messages/max_epoch.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Simple max value accumulator. When we dispatch messages we can store the max timestamp
+      # until which messages were dispatched by us. This allows us to quickly skip those messages
+      # during recovery, because we do know, they were dispatched.
+      class MaxEpoch
+        def initialize
+          @max = -1
+        end
+
+        # Updates epoch if bigger than current max
+        # @param new_max [Integer] potential new max epoch
+        def update(new_max)
+          return unless new_max
+          return unless new_max > @max
+
+          @max = new_max
+        end
+
+        # @return [Integer] max epoch recorded
+        def to_i
+          @max
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Proxy used to wrap the scheduled messages with the correct dispatch envelope.
+      # Each message that goes to the scheduler topic needs to have specific headers and other
+      # details that are required by the system so we know how and when to dispatch it.
+      #
+      # Each message that goes to the proxy topic needs to have a unique key. We inject those
+      # automatically unless user provides one in an envelope. Since we want to make sure, that
+      # the messages dispatched by the user all go to the same partition (if with same key), we
+      # inject a partition_key based on the user key or other details if present. That allows us
+      # to make sure, that they will always go to the same partition on our side.
+      #
+      # This wrapper validates the initial message that user wants to send in the future, as well
+      # as the envelope and specific requirements for a message to be send in the future
+      module Proxy
+        # General WaterDrop message contract. Before we envelop a message, we need to be certain
+        # it is correct, hence we use this contract.
+        MSG_CONTRACT = ::WaterDrop::Contracts::Message.new(
+          # Payload size is a subject to the target producer dispatch validation, so we set it
+          # to 100MB basically to ignore it here.
+          max_payload_size: 104_857_600
+        )
+
+        # Post-rebind contract to ensure, that user provided all needed details that would allow
+        # the system to operate correctly
+        POST_CONTRACT = Contracts::Message.new
+
+        # Attributes used to build a partition key for the schedules topic dispatch of a given
+        # message. We use this order as this order describes the priority of usage.
+        PARTITION_KEY_BASE_ATTRIBUTES = %i[
+          partition
+          partition_key
+        ].freeze
+
+        private_constant :MSG_CONTRACT, :POST_CONTRACT, :PARTITION_KEY_BASE_ATTRIBUTES
+
+        class << self
+          # Generates a schedule message envelope wrapping the original dispatch
+          #
+          # @param message [Hash] message hash of a message that would originally go to WaterDrop
+          #   producer directly.
+          # @param epoch [Integer] time in the future (or now) when dispatch this message in the
+          #   Unix epoch timestamp
+          # @param envelope [Hash] Special details that the envelop needs to have, like a unique
+          #   key. If unique key is not provided we build a random unique one and use a
+          #   partition_key based on the original message key (if present) to ensure that all
+          #   relevant messages are dispatched to the same topic partition.
+          # @return [Hash] dispatched message wrapped with an envelope
+          #
+          # @note This proxy does **not** inject the dispatched messages topic unless provided in
+          #   the envelope. That's because user can have multiple scheduled messages topics to
+          #   group outgoing messages, etc.
+          def schedule(message:, epoch:, envelope: {})
+            # We need to ensure that the message we want to proxy is fully legit. Otherwise, since
+            # we envelope details like target topic, we could end up having incorrect data to
+            # schedule
+            MSG_CONTRACT.validate!(message, WaterDrop::Errors::MessageInvalidError)
+
+            headers = (message[:headers] || {}).merge(
+              'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+              'schedule_target_epoch' => epoch.to_i.to_s,
+              'schedule_source_type' => 'schedule'
+            )
+
+            export(headers, message, :topic)
+            export(headers, message, :partition)
+            export(headers, message, :key)
+            export(headers, message, :partition_key)
+
+            proxy_message = {
+              payload: message[:payload],
+              headers: headers
+            }.merge(envelope)
+
+            enrich(proxy_message, message)
+
+            # Final validation to make sure all user provided extra data and what we have built
+            # complies with our requirements
+            POST_CONTRACT.validate!(proxy_message)
+            # After proxy specific validations we also ensure, that the final form is correct
+            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+
+            proxy_message
+          end
+
+          # Generates a tombstone message to cancel already scheduled message dispatch
+          # @param key [String] key used by the original message as a unique identifier
+          # @param envelope [Hash] Special details that can identify the message location like
+          #   topic and partition (if used) so the cancellation goes to the correct location.
+          # @return [Hash] cancellation message
+          #
+          # @note Technically it is a tombstone but we differentiate just for the sake of ability
+          #   to debug stuff if needed
+          def cancel(key:, envelope: {})
+            {
+              key: key,
+              payload: nil,
+              headers: {
+                'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+                'schedule_source_type' => 'cancel'
+              }
+            }.merge(envelope)
+          end
+
+          # Builds tombstone with the dispatched message details. Those details can be used
+          #   in Web UI, etc when analyzing dispatches.
+          # @param message [Karafka::Messages::Message] message we want to tombstone
+          #   topic and partition (if used) so the cancellation goes to the correct location.
+          def tombstone(message:)
+            {
+              key: message.key,
+              payload: nil,
+              topic: message.topic,
+              partition: message.partition,
+              headers: message.raw_headers.merge(
+                'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+                'schedule_source_type' => 'tombstone',
+                'schedule_source_offset' => message.offset.to_s
+              )
+            }
+          end
+
+          private
+
+          # Transfers the message key attributes into headers. Since we need to have our own
+          # envelope key and other details, we transfer the original message details into headers
+          # so we can re-use them when we dispatch the scheduled messages at an appropriate time
+          #
+          # @param headers [Hash] envelope headers to which we will add appropriate attribute
+          # @param message [Hash] original user message
+          # @param attribute [Symbol] attribute we're interested in exporting to headers
+          # @note Modifies headers in place
+          def export(headers, message, attribute)
+            return unless message.key?(attribute)
+
+            headers["schedule_target_#{attribute}"] = message.fetch(attribute).to_s
+          end
+
+          # Adds the key and (if applicable) partition key to ensure, that related messages that
+          # user wants to dispatch in the future, are all in the same topic partition.
+          # @param proxy_message [Hash] our message envelope
+          # @param message [Hash] user original message
+          # @note Modifies `proxy_message` in place
+          def enrich(proxy_message, message)
+            # If there is an envelope message key already, nothing needed
+            return if proxy_message.key?(:key)
+
+            proxy_message[:key] = "#{message[:topic]}-#{SecureRandom.uuid}"
+
+            PARTITION_KEY_BASE_ATTRIBUTES.each do |attribute|
+              next unless message.key?(attribute)
+              # Do not overwrite if explicitely set by the user
+              next if proxy_message.key?(attribute)
+
+              proxy_message[:partition_key] = message.fetch(attribute).to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/schema_validator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Validator that checks if we can process this scheduled message
+      # If we encounter message that has a schema version higher than our process is aware of
+      # we raise and error and do not process. This is to make sure we do not deal with messages
+      # that are not compatible in case of schema changes.
+      module SchemaValidator
+        class << self
+          # Check if we can work with this schema message and raise error if not.
+          #
+          # @param message [Karafka::Messages::Message]
+          def call(message)
+            message_version = message.headers['schedule_schema_version']
+
+            return if message_version <= ScheduledMessages::SCHEMA_VERSION
+
+            raise Errors::IncompatibleSchemaError, message_version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Serializers used to build payloads (if applicable) for dispatch
+      # @note We only deal with states payload. Other payloads are not ours but end users.
+      class Serializer
+        include ::Karafka::Core::Helpers::Time
+
+        # @param tracker [Tracker] tracker based on which we build the state
+        # @return [String] compressed payload with the state details
+        def state(tracker)
+          data = {
+            schema_version: ScheduledMessages::STATES_SCHEMA_VERSION,
+            dispatched_at: float_now,
+            state: tracker.state,
+            daily: tracker.daily
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        private
+
+        # @param hash [Hash] hash to cast to json
+        # @return [String] json hash
+        def serialize(hash)
+          hash.to_json
+        end
+
+        # Compresses the provided data
+        #
+        # @param data [String] data to compress
+        # @return [String] compressed data
+        def compress(data)
+          Zlib::Deflate.deflate(data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/setup/config.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Setup and config related recurring tasks components
+      module Setup
+        # Config for recurring tasks
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          setting(:consumer_class, default: Consumer)
+          setting(:group_id, default: 'karafka_scheduled_messages')
+
+          # By default we will run the scheduling every 15 seconds since we provide a minute-based
+          # precision. Can be increased when having dedicated processes to run this. Lower values
+          # mean more frequent execution on low-throughput topics meaning higher precision.
+          setting(:interval, default: 15_000)
+
+          # How many messages should be flush in one go from the dispatcher at most. If we have
+          # more messages to dispatch, they will be chunked.
+          setting(:flush_batch_size, default: 1_000)
+
+          # Producer to use. By default uses default Karafka producer.
+          setting(
+            :producer,
+            constructor: -> { ::Karafka.producer },
+            lazy: true
+          )
+
+          # Class we use to dispatch messages
+          setting(:dispatcher_class, default: Dispatcher)
+
+          # Postfix for the states topic to build the states based on the group name + postfix
+          setting(:states_postfix, default: '_states')
+
+          setting(:deserializers) do
+            # Deserializer for schedules messages to convert epochs
+            setting(:headers, default: Deserializers::Headers.new)
+            # Only applicable to states
+            setting(:payload, default: Deserializers::Payload.new)
+          end
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/state.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Represents the loading/bootstrapping state of the given topic partition
+      #
+      # Bootstrapping can be in the following states:
+      # - fresh - when we got an assignment but we did not load the schedule yet
+      # - loading - when we are in the process of bootstrapping the daily state and we consume
+      #   historical messages to build the needed schedules.
+      # - loaded - state in which we finished loading all the schedules and we can dispatch
+      #   messages when the time comes and we can process real-time incoming schedules and
+      #   changes to schedules as they appear in the stream.
+      class State
+        # @param loaded [nil, false, true] is the state loaded or not yet. `nil` indicates, it is
+        #   a fresh, pre-seek state.
+        def initialize(loaded = nil)
+          @loaded = loaded
+        end
+
+        # @return [Boolean] are we in a fresh, pre-bootstrap state
+        def fresh?
+          @loaded.nil?
+        end
+
+        # Marks the current state as fully loaded
+        def loaded!
+          @loaded = true
+        end
+
+        # @return [Boolean] are we in a loaded state
+        def loaded?
+          @loaded == true
+        end
+
+        # @return [String] current state string representation
+        def to_s
+          case @loaded
+          when nil
+            'fresh'
+          when false
+            'loading'
+          when true
+            'loaded'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/tracker.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Tracks basic state and metrics about schedules to be dispatched
+      #
+      # It provides accurate today dispatch taken from daily buffer and estimates for future days
+      class Tracker
+        # @return [Hash<String, Integer>]
+        attr_reader :daily
+
+        # @return [String] current state
+        attr_accessor :state
+
+        def initialize
+          @daily = Hash.new { |h, k| h[k] = 0 }
+          @created_at = Time.now.to_i
+        end
+
+        # Accurate (because coming from daily buffer) number of things to schedule
+        #
+        # @param sum [Integer]
+        def today=(sum)
+          @daily[epoch_to_date(@created_at)] = sum
+        end
+
+        # Tracks message dispatch
+        #
+        # It is only relevant for future days as for today we use accurate metrics from the daily
+        # buffer
+        #
+        # @param message [Karafka::Messages::Message] schedule message. Should **not** be a
+        #   tombstone message. Tombstone messages cancellations are not tracked because it would
+        #   drastically increase complexity. For given day we use the accurate counter and for
+        #   future days we use estimates.
+        def track(message)
+          epoch = message.headers['schedule_target_epoch']
+
+          @daily[epoch_to_date(epoch)] += 1
+        end
+
+        private
+
+        # @param epoch [Integer] epoch time
+        # @return [String] epoch matching date
+        def epoch_to_date(epoch)
+          Time.at(epoch).utc.to_date.to_s
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # This feature allows for proxying messages via a special topic that can dispatch them
+    # at a later time, hence scheduled messages. Such messages need to have a special format
+    # but aside from that they are regular Kafka messages.
+    #
+    # This work was conceptually inspired by the Go scheduler:
+    # https://github.com/etf1/kafka-message-scheduler though I did not look at the implementation
+    # itself. Just the concept of daily in-memory scheduling.
+    module ScheduledMessages
+      # Version of the schema we use for envelops in scheduled messages.
+      # We use it to detect any potential upgrades similar to other components of Karafka and to
+      # stop processing of incompatible versions
+      SCHEMA_VERSION = '1.0.0'
+
+      # Version of the states schema. Used to publish per partition simple aggregated metrics
+      # that can be used for schedules reporting
+      STATES_SCHEMA_VERSION = '1.0.0'
+
+      class << self
+        # Runs the `Proxy.call`
+        # @param kwargs [Hash] things requested by the proxy
+        # @return [Hash] message wrapped with the scheduled message envelope
+        def schedule(**kwargs)
+          Proxy.schedule(**kwargs)
+        end
+
+        # Generates a tombstone message to cancel given dispatch (if not yet happened)
+        # @param kwargs [Hash] things requested by the proxy
+        # @return [Hash] tombstone cancelling message
+        def cancel(**kwargs)
+          Proxy.cancel(**kwargs)
+        end
+
+        # Below are private APIs
+
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:scheduled_messages, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -181,6 +181,12 @@ module Karafka
           # overhead as this will happen only once per consumer lifetime
           consumer.messages = empty_messages
 
+          # Run the post-initialization hook for users that need to run some actions when consumer
+          # is built and ready (all basic state and info).
+          # Users should **not** overwrite the `#initialize` because it won't have all the needed
+          # data assigned yet.
+          consumer.on_initialized
+
           consumer
         end
       end

data/lib/karafka/processing/strategies/default.rb

@@ -31,6 +31,16 @@ module Karafka
           RUBY
         end
 
+        # Runs the post-creation, post-assignment code
+        # @note It runs in the listener loop. Should **not** be used for anything heavy or
+        #   with any potential errors. Mostly for initialization of states, etc.
+        def handle_initialized
+          Karafka.monitor.instrument('consumer.initialize', caller: self)
+          Karafka.monitor.instrument('consumer.initialized', caller: self) do
+            initialized
+          end
+        end
+
         # Marks message as consumed in an async way.
         #
         # @param message [Messages::Message] last successfully processed message.

data/lib/karafka/railtie.rb

@@ -65,26 +65,6 @@ if Karafka.rails?
         app.config.autoload_paths += %w[app/consumers]
       end
 
-      initializer 'karafka.release_active_record_connections' do
-        rails7plus = Rails.gem_version >= Gem::Version.new('7.0.0')
-
-        ActiveSupport.on_load(:active_record) do
-          ::Karafka::App.monitor.subscribe('worker.completed') do
-            # Always release the connection after processing is done. Otherwise thread may hang
-            # blocking the reload and further processing
-            # @see https://github.com/rails/rails/issues/44183
-            #
-            # The change technically happens in 7.1 but 7.0 already supports this so we can make
-            # a proper change for 7.0+
-            if rails7plus
-              ActiveRecord::Base.connection_handler.clear_active_connections!
-            else
-              ActiveRecord::Base.clear_active_connections!
-            end
-          end
-        end
-      end
-
       initializer 'karafka.require_karafka_boot_file' do |app|
         rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.9'
+  VERSION = '2.4.10'
 end