karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Consumer that coordinates scheduling of messages when the time comes
+      class Consumer < ::Karafka::BaseConsumer
+        # Prepares the initial state of all stateful components
+        def initialized
+          clear!
+          # Max epoch is always moving forward with the time. Never backwards, hence we do not
+          # reset it at all.
+          @max_epoch = MaxEpoch.new
+          @state = State.new(nil)
+        end
+
+        # Processes messages and runs dispatch (via tick) if needed
+        def consume
+          return if reload!
+
+          messages.each do |message|
+            SchemaValidator.call(message)
+            process_message(message)
+          end
+
+          @states_reporter.call
+
+          eofed if eofed?
+
+          # Unless given day data is fully loaded we should not dispatch any notifications nor
+          # should we mark messages.
+          return unless @state.loaded?
+
+          tick
+
+          # Despite the fact that we need to load the whole stream once a day we do mark.
+          # We mark as consumed for two main reasons:
+          #   - by marking we can indicate to Web UI and other monitoring tools that we have a
+          #     potential real lag with loading schedules in case there would be a lot of messages
+          #     added to the schedules topic
+          #   - we prevent a situation where there is no notion of this consumer group in the
+          #     reporting, allowing us to establish "presence"
+          mark_as_consumed(messages.last)
+        end
+
+        # Runs end of file operations
+        def eofed
+          return if reload!
+
+          # If end of the partition is reached, it always means all data is loaded
+          @state.loaded!
+          @states_reporter.call
+        end
+
+        # Performs periodic operations when no new data is provided to the topic partition
+        def tick
+          return if reload!
+
+          # We should not dispatch any data until the whole state is loaded. We need to make sure,
+          # that all tombstone events are loaded not to duplicate dispatches
+          return unless @state.loaded?
+
+          keys = []
+          epochs = []
+
+          # We first collect all the data for dispatch and then dispatch and **only** after
+          # dispatch that is sync is successful we remove those messages from the daily buffer
+          # and update the max epoch. Since only the dispatch itself is volatile and can crash
+          # with timeouts, etc, we need to be sure it wen through prior to deleting those messages
+          # from the daily buffer. That way we ensure the at least once delivery and in case of
+          # a transactional producer, exactly once delivery.
+          @daily_buffer.for_dispatch do |epoch, message|
+            epochs << epoch
+            keys << message.key
+            @dispatcher << message
+          end
+
+          @dispatcher.flush
+
+          @max_epoch.update(epochs.max)
+
+          keys.each { |key| @daily_buffer.delete(key) }
+
+          @states_reporter.call
+        end
+
+        private
+
+        # Takes each message and adds it to the daily accumulator if needed or performs other
+        # accumulator and time related per-message operations.
+        # @param message [Karafka::Messages::Message]
+        def process_message(message)
+          # If we started to receive messages younger than the moment we created the consumer for
+          # the given day, it means we have loaded all the history and we are no longer in the
+          # loading phase.
+          if message.timestamp.to_i > @today.created_at
+            @state.loaded!
+            tags.add(:state, @state.to_s)
+          end
+
+          # If this is a schedule message we need to check if this is for today. Tombstone events
+          # are always considered immediate as they indicate, that a message with a given key
+          # was already dispatched or that user decided not to dispatch and cancelled the dispatch
+          # via tombstone publishing.
+          if message.headers['schedule_source_type'] == 'schedule'
+            time = message.headers['schedule_target_epoch']
+
+            # Do not track historical below today as those will be reflected in the daily buffer
+            @tracker.track(message) if time >= @today.starts_at
+
+            if time > @today.ends_at || time < @max_epoch.to_i
+              # Clean the message immediately when not needed (won't be scheduled) to preserve
+              # memory
+              message.clean!
+
+              return
+            end
+          end
+
+          # Add to buffer all tombstones and messages for the same day
+          @daily_buffer << message
+        end
+
+        # Moves the offset back and re-seeks and reloads the current day not dispatched assignments
+        def reload!
+          # If this is a new assignment we always need to seek from beginning to load the data
+          if @state.fresh?
+            clear!
+            seek(0)
+
+            return true
+          end
+
+          # Unless state is loaded we do not do anything more because we're in the loading process
+          return false unless @state.loaded?
+
+          # If day has ended we reload and start new day with new schedules
+          if @today.ended?
+            clear!
+            seek(0)
+
+            return true
+          end
+
+          false
+        end
+
+        # Resets all buffers and states so we can start a new day with a clean slate
+        # We can fully recreate the dispatcher because any undispatched messages will be dispatched
+        # with the new day dispatcher after it is reloaded.
+        def clear!
+          @daily_buffer = DailyBuffer.new
+          @today = Day.new
+          @tracker = Tracker.new
+          @state = State.new(false)
+          @dispatcher = config.dispatcher_class.new(topic.name, partition)
+          @states_reporter = Helpers::IntervalRunner.new do
+            @tracker.today = @daily_buffer.size
+            @tracker.state = @state.to_s
+
+            @dispatcher.state(@tracker)
+          end
+
+          tags.add(:state, @state.to_s)
+        end
+
+        # @return [Karafka::Core::Configurable::Node] Schedules config node
+        def config
+          @config ||= Karafka::App.config.scheduled_messages
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/contracts/config.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Recurring Tasks related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:scheduled_messages) do
+            required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
+
+            # Do not allow to run more often than every second
+            required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+
+            required(:flush_batch_size) { |val| val.is_a?(Integer) && val.positive? }
+
+            required(:dispatcher_class) { |val| !val.nil? }
+
+            required(:group_id) do |val|
+              val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+            end
+
+            required(:states_postfix) do |val|
+              val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+            end
+
+            nested(:deserializers) do
+              required(:headers) { |val| !val.nil? }
+              required(:payload) { |val| !val.nil? }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/contracts/message.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      module Contracts
+        # Future message expected format.
+        #
+        # Our envelope always needs to comply with this format, otherwise we won't have enough
+        # details to be able to dispatch the message
+        class Message < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('scheduled_messages_message')
+          end
+
+          # Headers we expect in each message of type "schedule" that goes to our scheduled
+          # messages topic
+          EXPECTED_SCHEDULE_HEADERS = %w[
+            schedule_schema_version
+            schedule_target_epoch
+            schedule_source_type
+            schedule_target_topic
+          ].freeze
+
+          # Headers we expect in each message of type "cancel"
+          EXPECTED_CANCEL_HEADERS = %w[
+            schedule_schema_version
+            schedule_source_type
+          ].freeze
+
+          required(:key) { |val| val.is_a?(String) && val.size.positive? }
+
+          # Ensure that schedule has all correct keys and that others have other related data
+          required(:headers) do |val|
+            next false unless val.is_a?(Hash)
+
+            if val['schedule_source_type'] == 'message'
+              (val.keys & EXPECTED_SCHEDULE_HEADERS).size >= 4
+            else
+              (val.keys & EXPECTED_CANCEL_HEADERS).size >= 2
+            end
+          end
+
+          # Make sure, that schedule_target_epoch is not older than grace period behind us.
+          # While this is not ideal verification of scheduling stuff in past, at leats it will
+          # prevent user errors when they schedule at 0, etc
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            # Validate epoch only for schedules
+            next unless data[:headers]['schedule_source_type'] == 'schedule'
+
+            epoch_time = data[:headers].fetch('schedule_target_epoch').to_i
+
+            # We allow for small lag as those will be dispatched but we should prevent dispatching
+            # in the past in general as often it is a source of errors
+            next if epoch_time >= Time.now.to_i - 10
+
+            [[[:headers], :schedule_target_epoch_in_the_past]]
+          end
+
+          # Makes sure, that the target envelope topic we dispatch to is a scheduled messages topic
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            scheduled_topics = Karafka::App
+                               .routes
+                               .flat_map(&:topics)
+                               .flat_map(&:to_a)
+                               .select(&:scheduled_messages?)
+                               .map(&:name)
+
+            next if scheduled_topics.include?(data[:topic].to_s)
+
+            [[[:topic], :not_a_scheduled_messages_topic]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/daily_buffer.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Stores schedules for the current day and gives back those that should be dispatched
+      # We do not use min-heap implementation and just a regular hash because we want to be able
+      # to update the schedules based on the key as well as remove the schedules in case it would
+      # be cancelled. While removals could be implemented, updates with different timestamp would
+      # be more complex. At the moment a lookup of 8 640 000 messages (100 per second) takes
+      # up to 1.5 second, thus it is acceptable. Please ping me if you encounter performance
+      # issues with this naive implementation so it can be improved.
+      class DailyBuffer
+        def initialize
+          @accu = {}
+        end
+
+        # @return [Integer] number of elements to schedule today
+        def size
+          @accu.size
+        end
+
+        # Adds message to the buffer or removes the message from the buffer if it is a tombstone
+        # message for a given key
+        #
+        # @param message [Karafka::Messages::Message]
+        #
+        # @note Only messages for a given day should be added here.
+        def <<(message)
+          # Non schedule are only tombstones and cancellations
+          schedule = message.headers['schedule_source_type'] == 'schedule'
+
+          key = message.key
+
+          if schedule
+            epoch = message.headers['schedule_target_epoch']
+            @accu[key] = [epoch, message]
+          else
+            @accu.delete(key)
+          end
+        end
+
+        # Yields messages that should be dispatched (sent) to Kafka
+        #
+        # @yieldparam [Integer, Karafka::Messages::Message] epoch of the message and the message
+        #   itself
+        #
+        # @note We yield epoch alongside of the message so we do not have to extract it several
+        #   times later on. This simplifies the API
+        def for_dispatch
+          dispatch = Time.now.to_i
+
+          @accu.each_value do |epoch, message|
+            next unless epoch <= dispatch
+
+            yield(epoch, message)
+          end
+        end
+
+        # Removes given key from the accumulator
+        # @param key [String] key to remove
+        def delete(key)
+          @accu.delete(key)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/day.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Just a simple UTC day implementation.
+      # Since we operate on a scope of one day, this allows us to encapsulate when given day ends
+      class Day
+        # @return [Integer] utc timestamp when this day object was created. Keep in mind, that
+        #   this is **not** when the day started but when this object was created.
+        attr_reader :created_at
+        # @return [Integer] utc timestamp when this day ends (last second of day).
+        # Equal to 23:59:59.
+        attr_reader :ends_at
+        # @return [Integer] utc timestamp when this day starts. Equal to 00:00:00
+        attr_reader :starts_at
+
+        def initialize
+          @created_at = Time.now.to_i
+
+          time = Time.at(@created_at).utc
+
+          @starts_at = Time.utc(time.year, time.month, time.day).to_i
+          @ends_at = @starts_at + 86_399
+        end
+
+        # @return [Boolean] did the current day we operate on ended.
+        def ended?
+          @ends_at < Time.now.to_i
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Namespace for schedules data related deserializers.
+      module Deserializers
+        # Converts certain pieces of headers into their integer form for messages
+        class Headers
+          # @param metadata [Karafka::aMessages::Metadata]
+          # @return [Hash] headers
+          def call(metadata)
+            raw_headers = metadata.raw_headers
+
+            type = raw_headers.fetch('schedule_source_type')
+
+            # tombstone and cancellation events are not operable, thus we do not have to cast any
+            # of the headers pieces
+            return raw_headers unless type == 'schedule'
+
+            headers = raw_headers.dup
+            headers['schedule_target_epoch'] = headers['schedule_target_epoch'].to_i
+
+            # This attribute is optional, this is why we have to check for its existence
+            if headers.key?('schedule_target_partition')
+              headers['schedule_target_partition'] = headers['schedule_target_partition'].to_i
+            end
+
+            headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      module Deserializers
+        # States payload deserializer
+        # We only deserialize states data and never anything else. Other payloads are the payloads
+        # we are expected to proxy, thus there is no need to deserialize them in any context.
+        # Their appropriate target topics should have expected deserializers
+        class Payload
+          # @param message [::Karafka::Messages::Message]
+          # @return [Hash] deserialized data
+          def call(message)
+            ::JSON.parse(
+              Zlib::Inflate.inflate(message.raw_payload),
+              symbolize_names: true
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/dispatcher.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Dispatcher responsible for dispatching the messages to appropriate target topics and for
+      # dispatching other messages. All messages (aside from the once users dispatch with the
+      # envelope) are sent via this dispatcher.
+      #
+      # Messages are buffered and dispatched in batches to improve dispatch performance.
+      class Dispatcher
+        # @return [Array<Hash>] buffer with message hashes for dispatch
+        attr_reader :buffer
+
+        # @param topic [String] consumed topic name
+        # @param partition [Integer] consumed partition
+        def initialize(topic, partition)
+          @topic = topic
+          @partition = partition
+          @buffer = []
+          @serializer = Serializer.new
+        end
+
+        # Prepares the scheduled message to the dispatch to the target topic. Extracts all the
+        # "schedule_" details and prepares it, so the dispatched message goes with the expected
+        # attributes to the desired location. Alongside of that it actually builds 2
+        # (1 if logs off) messages: tombstone event matching the schedule so it is no longer valid
+        # and the log message that has the same data as the dispatched message. Helpful when
+        # debugging.
+        #
+        # @param message [Karafka::Messages::Message] message from the schedules topic.
+        #
+        # @note This method adds the message to the buffer, does **not** dispatch it.
+        # @note It also produces needed tombstone event as well as an audit log message
+        def <<(message)
+          target_headers = message.raw_headers.merge(
+            'schedule_source_topic' => @topic,
+            'schedule_source_partition' => @partition.to_s,
+            'schedule_source_offset' => message.offset.to_s,
+            'schedule_source_key' => message.key
+          ).compact
+
+          target = {
+            payload: message.raw_payload,
+            headers: target_headers
+          }
+
+          extract(target, message.headers, :topic)
+          extract(target, message.headers, :partition)
+          extract(target, message.headers, :key)
+          extract(target, message.headers, :partition_key)
+
+          @buffer << target
+
+          # Tombstone message so this schedule is no longer in use and gets removed from Kafka by
+          # Kafka itself during compacting. It will not cancel it because already dispatched but
+          # will cause it not to be sent again and will be marked as dispatched.
+          @buffer << Proxy.tombstone(message: message)
+        end
+
+        # Builds and dispatches the state report message with schedules details
+        #
+        # @param tracker [Tracker]
+        #
+        # @note This is dispatched async because it's just a statistical metric.
+        def state(tracker)
+          config.producer.produce_async(
+            topic: "#{@topic}#{config.states_postfix}",
+            payload: @serializer.state(tracker),
+            key: 'state',
+            partition: @partition,
+            headers: { 'zlib' => 'true' }
+          )
+        end
+
+        # Sends all messages to Kafka in a sync way.
+        # We use sync with batches to prevent overloading.
+        # When transactional producer in use, this will be wrapped in a transaction automatically.
+        def flush
+          until @buffer.empty?
+            config.producer.produce_many_sync(
+              # We can remove this prior to the dispatch because we only evict messages from the
+              # daily buffer once dispatch is successful
+              @buffer.shift(config.flush_batch_size)
+            )
+          end
+        end
+
+        private
+
+        # @return [Karafka::Core::Configurable::Node] scheduled messages config node
+        def config
+          @config ||= Karafka::App.config.scheduled_messages
+        end
+
+        # Extracts and copies the future attribute to a proper place in the target message.
+        #
+        # @param target [Hash]
+        # @param headers [Hash]
+        # @param attribute [Symbol]
+        def extract(target, headers, attribute)
+          schedule_attribute = "schedule_target_#{attribute}"
+
+          return unless headers.key?(schedule_attribute)
+
+          target[attribute] = headers[schedule_attribute]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/errors.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Scheduled Messages related errors
+      module Errors
+        # Base for all the recurring tasks errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when the scheduled messages processor is older than the messages we started to
+        # receive. In such cases we stop because there may be schema changes.
+        IncompatibleSchemaError = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/max_epoch.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Simple max value accumulator. When we dispatch messages we can store the max timestamp
+      # until which messages were dispatched by us. This allows us to quickly skip those messages
+      # during recovery, because we do know, they were dispatched.
+      class MaxEpoch
+        def initialize
+          @max = -1
+        end
+
+        # Updates epoch if bigger than current max
+        # @param new_max [Integer] potential new max epoch
+        def update(new_max)
+          return unless new_max
+          return unless new_max > @max
+
+          @max = new_max
+        end
+
+        # @return [Integer] max epoch recorded
+        def to_i
+          @max
+        end
+      end
+    end
+  end
+end