karafka 2.4.8 → 2.4.10

data/lib/karafka/pro/scheduled_messages/daily_buffer.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Stores schedules for the current day and gives back those that should be dispatched
+      # We do not use min-heap implementation and just a regular hash because we want to be able
+      # to update the schedules based on the key as well as remove the schedules in case it would
+      # be cancelled. While removals could be implemented, updates with different timestamp would
+      # be more complex. At the moment a lookup of 8 640 000 messages (100 per second) takes
+      # up to 1.5 second, thus it is acceptable. Please ping me if you encounter performance
+      # issues with this naive implementation so it can be improved.
+      class DailyBuffer
+        def initialize
+          @accu = {}
+        end
+
+        # @return [Integer] number of elements to schedule today
+        def size
+          @accu.size
+        end
+
+        # Adds message to the buffer or removes the message from the buffer if it is a tombstone
+        # message for a given key
+        #
+        # @param message [Karafka::Messages::Message]
+        #
+        # @note Only messages for a given day should be added here.
+        def <<(message)
+          # Non schedule are only tombstones and cancellations
+          schedule = message.headers['schedule_source_type'] == 'schedule'
+
+          key = message.key
+
+          if schedule
+            epoch = message.headers['schedule_target_epoch']
+            @accu[key] = [epoch, message]
+          else
+            @accu.delete(key)
+          end
+        end
+
+        # Yields messages that should be dispatched (sent) to Kafka
+        #
+        # @yieldparam [Integer, Karafka::Messages::Message] epoch of the message and the message
+        #   itself
+        #
+        # @note We yield epoch alongside of the message so we do not have to extract it several
+        #   times later on. This simplifies the API
+        def for_dispatch
+          dispatch = Time.now.to_i
+
+          @accu.each_value do |epoch, message|
+            next unless epoch <= dispatch
+
+            yield(epoch, message)
+          end
+        end
+
+        # Removes given key from the accumulator
+        # @param key [String] key to remove
+        def delete(key)
+          @accu.delete(key)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/day.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Just a simple UTC day implementation.
+      # Since we operate on a scope of one day, this allows us to encapsulate when given day ends
+      class Day
+        # @return [Integer] utc timestamp when this day object was created. Keep in mind, that
+        #   this is **not** when the day started but when this object was created.
+        attr_reader :created_at
+        # @return [Integer] utc timestamp when this day ends (last second of day).
+        # Equal to 23:59:59.
+        attr_reader :ends_at
+        # @return [Integer] utc timestamp when this day starts. Equal to 00:00:00
+        attr_reader :starts_at
+
+        def initialize
+          @created_at = Time.now.to_i
+
+          time = Time.at(@created_at)
+
+          @starts_at = Time.utc(time.year, time.month, time.day).to_i
+          @ends_at = @starts_at + 86_399
+        end
+
+        # @return [Boolean] did the current day we operate on ended.
+        def ended?
+          @ends_at < Time.now.to_i
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Namespace for schedules data related deserializers.
+      module Deserializers
+        # Converts certain pieces of headers into their integer form for messages
+        class Headers
+          # @param metadata [Karafka::aMessages::Metadata]
+          # @return [Hash] headers
+          def call(metadata)
+            raw_headers = metadata.raw_headers
+
+            type = raw_headers.fetch('schedule_source_type')
+
+            # tombstone and cancellation events are not operable, thus we do not have to cast any
+            # of the headers pieces
+            return raw_headers unless type == 'schedule'
+
+            headers = raw_headers.dup
+            headers['schedule_target_epoch'] = headers['schedule_target_epoch'].to_i
+
+            # This attribute is optional, this is why we have to check for its existence
+            if headers.key?('schedule_target_partition')
+              headers['schedule_target_partition'] = headers['schedule_target_partition'].to_i
+            end
+
+            headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      module Deserializers
+        # States payload deserializer
+        # We only deserialize states data and never anything else. Other payloads are the payloads
+        # we are expected to proxy, thus there is no need to deserialize them in any context.
+        # Their appropriate target topics should have expected deserializers
+        class Payload
+          # @param message [::Karafka::Messages::Message]
+          # @return [Hash] deserialized data
+          def call(message)
+            ::JSON.parse(
+              Zlib::Inflate.inflate(message.raw_payload),
+              symbolize_names: true
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/dispatcher.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Dispatcher responsible for dispatching the messages to appropriate target topics and for
+      # dispatching other messages. All messages (aside from the once users dispatch with the
+      # envelope) are sent via this dispatcher.
+      #
+      # Messages are buffered and dispatched in batches to improve dispatch performance.
+      class Dispatcher
+        # @return [Array<Hash>] buffer with message hashes for dispatch
+        attr_reader :buffer
+
+        # @param topic [String] consumed topic name
+        # @param partition [Integer] consumed partition
+        def initialize(topic, partition)
+          @topic = topic
+          @partition = partition
+          @buffer = []
+          @serializer = Serializer.new
+        end
+
+        # Prepares the scheduled message to the dispatch to the target topic. Extracts all the
+        # "schedule_" details and prepares it, so the dispatched message goes with the expected
+        # attributes to the desired location. Alongside of that it actually builds 2
+        # (1 if logs off) messages: tombstone event matching the schedule so it is no longer valid
+        # and the log message that has the same data as the dispatched message. Helpful when
+        # debugging.
+        #
+        # @param message [Karafka::Messages::Message] message from the schedules topic.
+        #
+        # @note This method adds the message to the buffer, does **not** dispatch it.
+        # @note It also produces needed tombstone event as well as an audit log message
+        def <<(message)
+          target_headers = message.raw_headers.merge(
+            'schedule_source_topic' => @topic,
+            'schedule_source_partition' => @partition.to_s,
+            'schedule_source_offset' => message.offset.to_s,
+            'schedule_source_key' => message.key
+          ).compact
+
+          target = {
+            payload: message.raw_payload,
+            headers: target_headers
+          }
+
+          extract(target, message.headers, :topic)
+          extract(target, message.headers, :partition)
+          extract(target, message.headers, :key)
+          extract(target, message.headers, :partition_key)
+
+          @buffer << target
+
+          # Tombstone message so this schedule is no longer in use and gets removed from Kafka by
+          # Kafka itself during compacting. It will not cancel it because already dispatched but
+          # will cause it not to be sent again and will be marked as dispatched.
+          @buffer << Proxy.tombstone(message: message)
+        end
+
+        # Builds and dispatches the state report message with schedules details
+        #
+        # @param tracker [Tracker]
+        #
+        # @note This is dispatched async because it's just a statistical metric.
+        def state(tracker)
+          config.producer.produce_async(
+            topic: "#{@topic}#{config.states_postfix}",
+            payload: @serializer.state(tracker),
+            key: 'state',
+            partition: @partition,
+            headers: { 'zlib' => 'true' }
+          )
+        end
+
+        # Sends all messages to Kafka in a sync way.
+        # We use sync with batches to prevent overloading.
+        # When transactional producer in use, this will be wrapped in a transaction automatically.
+        def flush
+          until @buffer.empty?
+            config.producer.produce_many_sync(
+              # We can remove this prior to the dispatch because we only evict messages from the
+              # daily buffer once dispatch is successful
+              @buffer.shift(config.flush_batch_size)
+            )
+          end
+        end
+
+        private
+
+        # @return [Karafka::Core::Configurable::Node] scheduled messages config node
+        def config
+          @config ||= Karafka::App.config.scheduled_messages
+        end
+
+        # Extracts and copies the future attribute to a proper place in the target message.
+        #
+        # @param target [Hash]
+        # @param headers [Hash]
+        # @param attribute [Symbol]
+        def extract(target, headers, attribute)
+          schedule_attribute = "schedule_target_#{attribute}"
+
+          return unless headers.key?(schedule_attribute)
+
+          target[attribute] = headers[schedule_attribute]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/errors.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Scheduled Messages related errors
+      module Errors
+        # Base for all the recurring tasks errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when the scheduled messages processor is older than the messages we started to
+        # receive. In such cases we stop because there may be schema changes.
+        IncompatibleSchemaError = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/max_epoch.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Simple max value accumulator. When we dispatch messages we can store the max timestamp
+      # until which messages were dispatched by us. This allows us to quickly skip those messages
+      # during recovery, because we do know, they were dispatched.
+      class MaxEpoch
+        def initialize
+          @max = -1
+        end
+
+        # Updates epoch if bigger than current max
+        # @param new_max [Integer] potential new max epoch
+        def update(new_max)
+          return unless new_max
+          return unless new_max > @max
+
+          @max = new_max
+        end
+
+        # @return [Integer] max epoch recorded
+        def to_i
+          @max
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Proxy used to wrap the scheduled messages with the correct dispatch envelope.
+      # Each message that goes to the scheduler topic needs to have specific headers and other
+      # details that are required by the system so we know how and when to dispatch it.
+      #
+      # Each message that goes to the proxy topic needs to have a unique key. We inject those
+      # automatically unless user provides one in an envelope. Since we want to make sure, that
+      # the messages dispatched by the user all go to the same partition (if with same key), we
+      # inject a partition_key based on the user key or other details if present. That allows us
+      # to make sure, that they will always go to the same partition on our side.
+      #
+      # This wrapper validates the initial message that user wants to send in the future, as well
+      # as the envelope and specific requirements for a message to be send in the future
+      module Proxy
+        # General WaterDrop message contract. Before we envelop a message, we need to be certain
+        # it is correct, hence we use this contract.
+        MSG_CONTRACT = ::WaterDrop::Contracts::Message.new(
+          # Payload size is a subject to the target producer dispatch validation, so we set it
+          # to 100MB basically to ignore it here.
+          max_payload_size: 104_857_600
+        )
+
+        # Post-rebind contract to ensure, that user provided all needed details that would allow
+        # the system to operate correctly
+        POST_CONTRACT = Contracts::Message.new
+
+        # Attributes used to build a partition key for the schedules topic dispatch of a given
+        # message. We use this order as this order describes the priority of usage.
+        PARTITION_KEY_BASE_ATTRIBUTES = %i[
+          partition
+          partition_key
+        ].freeze
+
+        private_constant :MSG_CONTRACT, :POST_CONTRACT, :PARTITION_KEY_BASE_ATTRIBUTES
+
+        class << self
+          # Generates a schedule message envelope wrapping the original dispatch
+          #
+          # @param message [Hash] message hash of a message that would originally go to WaterDrop
+          #   producer directly.
+          # @param epoch [Integer] time in the future (or now) when dispatch this message in the
+          #   Unix epoch timestamp
+          # @param envelope [Hash] Special details that the envelop needs to have, like a unique
+          #   key. If unique key is not provided we build a random unique one and use a
+          #   partition_key based on the original message key (if present) to ensure that all
+          #   relevant messages are dispatched to the same topic partition.
+          # @return [Hash] dispatched message wrapped with an envelope
+          #
+          # @note This proxy does **not** inject the dispatched messages topic unless provided in
+          #   the envelope. That's because user can have multiple scheduled messages topics to
+          #   group outgoing messages, etc.
+          def schedule(message:, epoch:, envelope: {})
+            # We need to ensure that the message we want to proxy is fully legit. Otherwise, since
+            # we envelope details like target topic, we could end up having incorrect data to
+            # schedule
+            MSG_CONTRACT.validate!(message, WaterDrop::Errors::MessageInvalidError)
+
+            headers = (message[:headers] || {}).merge(
+              'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+              'schedule_target_epoch' => epoch.to_i.to_s,
+              'schedule_source_type' => 'schedule'
+            )
+
+            export(headers, message, :topic)
+            export(headers, message, :partition)
+            export(headers, message, :key)
+            export(headers, message, :partition_key)
+
+            proxy_message = {
+              payload: message[:payload],
+              headers: headers
+            }.merge(envelope)
+
+            enrich(proxy_message, message)
+
+            # Final validation to make sure all user provided extra data and what we have built
+            # complies with our requirements
+            POST_CONTRACT.validate!(proxy_message)
+            # After proxy specific validations we also ensure, that the final form is correct
+            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+
+            proxy_message
+          end
+
+          # Generates a tombstone message to cancel already scheduled message dispatch
+          # @param key [String] key used by the original message as a unique identifier
+          # @param envelope [Hash] Special details that can identify the message location like
+          #   topic and partition (if used) so the cancellation goes to the correct location.
+          # @return [Hash] cancellation message
+          #
+          # @note Technically it is a tombstone but we differentiate just for the sake of ability
+          #   to debug stuff if needed
+          def cancel(key:, envelope: {})
+            {
+              key: key,
+              payload: nil,
+              headers: {
+                'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+                'schedule_source_type' => 'cancel'
+              }
+            }.merge(envelope)
+          end
+
+          # Builds tombstone with the dispatched message details. Those details can be used
+          #   in Web UI, etc when analyzing dispatches.
+          # @param message [Karafka::Messages::Message] message we want to tombstone
+          #   topic and partition (if used) so the cancellation goes to the correct location.
+          def tombstone(message:)
+            {
+              key: message.key,
+              payload: nil,
+              topic: message.topic,
+              partition: message.partition,
+              headers: message.raw_headers.merge(
+                'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+                'schedule_source_type' => 'tombstone',
+                'schedule_source_offset' => message.offset.to_s
+              )
+            }
+          end
+
+          private
+
+          # Transfers the message key attributes into headers. Since we need to have our own
+          # envelope key and other details, we transfer the original message details into headers
+          # so we can re-use them when we dispatch the scheduled messages at an appropriate time
+          #
+          # @param headers [Hash] envelope headers to which we will add appropriate attribute
+          # @param message [Hash] original user message
+          # @param attribute [Symbol] attribute we're interested in exporting to headers
+          # @note Modifies headers in place
+          def export(headers, message, attribute)
+            return unless message.key?(attribute)
+
+            headers["schedule_target_#{attribute}"] = message.fetch(attribute).to_s
+          end
+
+          # Adds the key and (if applicable) partition key to ensure, that related messages that
+          # user wants to dispatch in the future, are all in the same topic partition.
+          # @param proxy_message [Hash] our message envelope
+          # @param message [Hash] user original message
+          # @note Modifies `proxy_message` in place
+          def enrich(proxy_message, message)
+            # If there is an envelope message key already, nothing needed
+            return if proxy_message.key?(:key)
+
+            proxy_message[:key] = "#{message[:topic]}-#{SecureRandom.uuid}"
+
+            PARTITION_KEY_BASE_ATTRIBUTES.each do |attribute|
+              next unless message.key?(attribute)
+              # Do not overwrite if explicitely set by the user
+              next if proxy_message.key?(attribute)
+
+              proxy_message[:partition_key] = message.fetch(attribute).to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/schema_validator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Validator that checks if we can process this scheduled message
+      # If we encounter message that has a schema version higher than our process is aware of
+      # we raise and error and do not process. This is to make sure we do not deal with messages
+      # that are not compatible in case of schema changes.
+      module SchemaValidator
+        class << self
+          # Check if we can work with this schema message and raise error if not.
+          #
+          # @param message [Karafka::Messages::Message]
+          def call(message)
+            message_version = message.headers['schedule_schema_version']
+
+            return if message_version <= ScheduledMessages::SCHEMA_VERSION
+
+            raise Errors::IncompatibleSchemaError, message_version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Serializers used to build payloads (if applicable) for dispatch
+      # @note We only deal with states payload. Other payloads are not ours but end users.
+      class Serializer
+        include ::Karafka::Core::Helpers::Time
+
+        # @param tracker [Tracker] tracker based on which we build the state
+        # @return [String] compressed payload with the state details
+        def state(tracker)
+          data = {
+            schema_version: ScheduledMessages::STATES_SCHEMA_VERSION,
+            dispatched_at: float_now,
+            state: tracker.state,
+            daily: tracker.daily
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        private
+
+        # @param hash [Hash] hash to cast to json
+        # @return [String] json hash
+        def serialize(hash)
+          hash.to_json
+        end
+
+        # Compresses the provided data
+        #
+        # @param data [String] data to compress
+        # @return [String] compressed data
+        def compress(data)
+          Zlib::Deflate.deflate(data)
+        end
+      end
+    end
+  end
+end