karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Proxy used to wrap the scheduled messages with the correct dispatch envelope.
+      # Each message that goes to the scheduler topic needs to have specific headers and other
+      # details that are required by the system so we know how and when to dispatch it.
+      #
+      # Each message that goes to the proxy topic needs to have a unique key. We inject those
+      # automatically unless user provides one in an envelope. Since we want to make sure, that
+      # the messages dispatched by the user all go to the same partition (if with same key), we
+      # inject a partition_key based on the user key or other details if present. That allows us
+      # to make sure, that they will always go to the same partition on our side.
+      #
+      # This wrapper validates the initial message that user wants to send in the future, as well
+      # as the envelope and specific requirements for a message to be send in the future
+      module Proxy
+        # General WaterDrop message contract. Before we envelop a message, we need to be certain
+        # it is correct, hence we use this contract.
+        MSG_CONTRACT = ::WaterDrop::Contracts::Message.new(
+          # Payload size is a subject to the target producer dispatch validation, so we set it
+          # to 100MB basically to ignore it here.
+          max_payload_size: 104_857_600
+        )
+
+        # Post-rebind contract to ensure, that user provided all needed details that would allow
+        # the system to operate correctly
+        POST_CONTRACT = Contracts::Message.new
+
+        # Attributes used to build a partition key for the schedules topic dispatch of a given
+        # message. We use this order as this order describes the priority of usage.
+        PARTITION_KEY_BASE_ATTRIBUTES = %i[
+          partition
+          partition_key
+        ].freeze
+
+        private_constant :MSG_CONTRACT, :POST_CONTRACT, :PARTITION_KEY_BASE_ATTRIBUTES
+
+        class << self
+          # Generates a schedule message envelope wrapping the original dispatch
+          #
+          # @param message [Hash] message hash of a message that would originally go to WaterDrop
+          #   producer directly.
+          # @param epoch [Integer] time in the future (or now) when dispatch this message in the
+          #   Unix epoch timestamp
+          # @param envelope [Hash] Special details that the envelop needs to have, like a unique
+          #   key. If unique key is not provided we build a random unique one and use a
+          #   partition_key based on the original message key (if present) to ensure that all
+          #   relevant messages are dispatched to the same topic partition.
+          # @return [Hash] dispatched message wrapped with an envelope
+          #
+          # @note This proxy does **not** inject the dispatched messages topic unless provided in
+          #   the envelope. That's because user can have multiple scheduled messages topics to
+          #   group outgoing messages, etc.
+          def schedule(message:, epoch:, envelope: {})
+            # We need to ensure that the message we want to proxy is fully legit. Otherwise, since
+            # we envelope details like target topic, we could end up having incorrect data to
+            # schedule
+            MSG_CONTRACT.validate!(message, WaterDrop::Errors::MessageInvalidError)
+
+            headers = (message[:headers] || {}).merge(
+              'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+              'schedule_target_epoch' => epoch.to_i.to_s,
+              'schedule_source_type' => 'schedule'
+            )
+
+            export(headers, message, :topic)
+            export(headers, message, :partition)
+            export(headers, message, :key)
+            export(headers, message, :partition_key)
+
+            proxy_message = {
+              payload: message[:payload],
+              headers: headers
+            }.merge(envelope)
+
+            enrich(proxy_message, message)
+            validate!(proxy_message)
+
+            proxy_message
+          end
+
+          # Generates a tombstone message to cancel already scheduled message dispatch
+          # @param key [String] key used by the original message as a unique identifier
+          # @param envelope [Hash] Special details that can identify the message location like
+          #   topic and partition (if used) so the cancellation goes to the correct location.
+          # @return [Hash] cancellation message
+          #
+          # @note Technically it is a tombstone but we differentiate just for the sake of ability
+          #   to debug stuff if needed
+          def cancel(key:, envelope: {})
+            proxy_message = {
+              key: key,
+              payload: nil,
+              headers: {
+                'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+                'schedule_source_type' => 'cancel'
+              }
+            }.merge(envelope)
+
+            # Ensure user provided envelope is with all expected details
+            validate!(proxy_message)
+
+            proxy_message
+          end
+
+          # Builds tombstone with the dispatched message details. Those details can be used
+          #   in Web UI, etc when analyzing dispatches.
+          # @param message [Karafka::Messages::Message] message we want to tombstone
+          #   topic and partition (if used) so the cancellation goes to the correct location.
+          def tombstone(message:)
+            {
+              key: message.key,
+              payload: nil,
+              topic: message.topic,
+              partition: message.partition,
+              headers: message.raw_headers.merge(
+                'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
+                'schedule_source_type' => 'tombstone',
+                'schedule_source_offset' => message.offset.to_s
+              )
+            }
+          end
+
+          private
+
+          # Transfers the message key attributes into headers. Since we need to have our own
+          # envelope key and other details, we transfer the original message details into headers
+          # so we can re-use them when we dispatch the scheduled messages at an appropriate time
+          #
+          # @param headers [Hash] envelope headers to which we will add appropriate attribute
+          # @param message [Hash] original user message
+          # @param attribute [Symbol] attribute we're interested in exporting to headers
+          # @note Modifies headers in place
+          def export(headers, message, attribute)
+            return unless message.key?(attribute)
+
+            headers["schedule_target_#{attribute}"] = message.fetch(attribute).to_s
+          end
+
+          # Adds the key and (if applicable) partition key to ensure, that related messages that
+          # user wants to dispatch in the future, are all in the same topic partition.
+          # @param proxy_message [Hash] our message envelope
+          # @param message [Hash] user original message
+          # @note Modifies `proxy_message` in place
+          def enrich(proxy_message, message)
+            # If there is an envelope message key already, nothing needed
+            return if proxy_message.key?(:key)
+
+            proxy_message[:key] = "#{message[:topic]}-#{SecureRandom.uuid}"
+
+            PARTITION_KEY_BASE_ATTRIBUTES.each do |attribute|
+              next unless message.key?(attribute)
+              # Do not overwrite if explicitely set by the user
+              next if proxy_message.key?(attribute)
+
+              proxy_message[:partition_key] = message.fetch(attribute).to_s
+            end
+          end
+
+          # Final validations to make sure all user provided extra data and what we have built
+          # complies with our requirements
+          # @param proxy_message [Hash] our message envelope
+          def validate!(proxy_message)
+            POST_CONTRACT.validate!(proxy_message)
+            # After proxy specific validations we also ensure, that the final form is correct
+            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/schema_validator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Validator that checks if we can process this scheduled message
+      # If we encounter message that has a schema version higher than our process is aware of
+      # we raise and error and do not process. This is to make sure we do not deal with messages
+      # that are not compatible in case of schema changes.
+      module SchemaValidator
+        class << self
+          # Check if we can work with this schema message and raise error if not.
+          #
+          # @param message [Karafka::Messages::Message]
+          def call(message)
+            message_version = message.headers['schedule_schema_version']
+
+            return if message_version <= ScheduledMessages::SCHEMA_VERSION
+
+            raise Errors::IncompatibleSchemaError, message_version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Serializers used to build payloads (if applicable) for dispatch
+      # @note We only deal with states payload. Other payloads are not ours but end users.
+      class Serializer
+        include ::Karafka::Core::Helpers::Time
+
+        # @param tracker [Tracker] tracker based on which we build the state
+        # @return [String] compressed payload with the state details
+        def state(tracker)
+          data = {
+            schema_version: ScheduledMessages::STATES_SCHEMA_VERSION,
+            dispatched_at: float_now,
+            state: tracker.state,
+            daily: tracker.daily
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        private
+
+        # @param hash [Hash] hash to cast to json
+        # @return [String] json hash
+        def serialize(hash)
+          hash.to_json
+        end
+
+        # Compresses the provided data
+        #
+        # @param data [String] data to compress
+        # @return [String] compressed data
+        def compress(data)
+          Zlib::Deflate.deflate(data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/setup/config.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Setup and config related recurring tasks components
+      module Setup
+        # Config for recurring tasks
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          setting(:consumer_class, default: Consumer)
+          setting(:group_id, default: 'karafka_scheduled_messages')
+
+          # By default we will run the scheduling every 15 seconds since we provide a minute-based
+          # precision. Can be increased when having dedicated processes to run this. Lower values
+          # mean more frequent execution on low-throughput topics meaning higher precision.
+          setting(:interval, default: 15_000)
+
+          # How many messages should be flush in one go from the dispatcher at most. If we have
+          # more messages to dispatch, they will be chunked.
+          setting(:flush_batch_size, default: 1_000)
+
+          # Producer to use. By default uses default Karafka producer.
+          setting(
+            :producer,
+            constructor: -> { ::Karafka.producer },
+            lazy: true
+          )
+
+          # Class we use to dispatch messages
+          setting(:dispatcher_class, default: Dispatcher)
+
+          # Postfix for the states topic to build the states based on the group name + postfix
+          setting(:states_postfix, default: '_states')
+
+          setting(:deserializers) do
+            # Deserializer for schedules messages to convert epochs
+            setting(:headers, default: Deserializers::Headers.new)
+            # Only applicable to states
+            setting(:payload, default: Deserializers::Payload.new)
+          end
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/state.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Represents the loading/bootstrapping state of the given topic partition
+      #
+      # Bootstrapping can be in the following states:
+      # - fresh - when we got an assignment but we did not load the schedule yet
+      # - loading - when we are in the process of bootstrapping the daily state and we consume
+      #   historical messages to build the needed schedules.
+      # - loaded - state in which we finished loading all the schedules and we can dispatch
+      #   messages when the time comes and we can process real-time incoming schedules and
+      #   changes to schedules as they appear in the stream.
+      class State
+        # @param loaded [nil, false, true] is the state loaded or not yet. `nil` indicates, it is
+        #   a fresh, pre-seek state.
+        def initialize(loaded = nil)
+          @loaded = loaded
+        end
+
+        # @return [Boolean] are we in a fresh, pre-bootstrap state
+        def fresh?
+          @loaded.nil?
+        end
+
+        # Marks the current state as fully loaded
+        def loaded!
+          @loaded = true
+        end
+
+        # @return [Boolean] are we in a loaded state
+        def loaded?
+          @loaded == true
+        end
+
+        # @return [String] current state string representation
+        def to_s
+          case @loaded
+          when nil
+            'fresh'
+          when false
+            'loading'
+          when true
+            'loaded'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/tracker.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module ScheduledMessages
+      # Tracks basic state and metrics about schedules to be dispatched
+      #
+      # It provides accurate today dispatch taken from daily buffer and estimates for future days
+      class Tracker
+        # @return [Hash<String, Integer>]
+        attr_reader :daily
+
+        # @return [String] current state
+        attr_accessor :state
+
+        def initialize
+          @daily = Hash.new { |h, k| h[k] = 0 }
+          @created_at = Time.now.to_i
+        end
+
+        # Accurate (because coming from daily buffer) number of things to schedule
+        #
+        # @param sum [Integer]
+        def today=(sum)
+          @daily[epoch_to_date(@created_at)] = sum
+        end
+
+        # Tracks message dispatch
+        #
+        # It is only relevant for future days as for today we use accurate metrics from the daily
+        # buffer
+        #
+        # @param message [Karafka::Messages::Message] schedule message. Should **not** be a
+        #   tombstone message. Tombstone messages cancellations are not tracked because it would
+        #   drastically increase complexity. For given day we use the accurate counter and for
+        #   future days we use estimates.
+        def track(message)
+          epoch = message.headers['schedule_target_epoch']
+
+          @daily[epoch_to_date(epoch)] += 1
+        end
+
+        private
+
+        # @param epoch [Integer] epoch time
+        # @return [String] epoch matching date
+        def epoch_to_date(epoch)
+          Time.at(epoch).utc.to_date.to_s
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    # This feature allows for proxying messages via a special topic that can dispatch them
+    # at a later time, hence scheduled messages. Such messages need to have a special format
+    # but aside from that they are regular Kafka messages.
+    #
+    # This work was conceptually inspired by the Go scheduler:
+    # https://github.com/etf1/kafka-message-scheduler though I did not look at the implementation
+    # itself. Just the concept of daily in-memory scheduling.
+    module ScheduledMessages
+      # Version of the schema we use for envelops in scheduled messages.
+      # We use it to detect any potential upgrades similar to other components of Karafka and to
+      # stop processing of incompatible versions
+      SCHEMA_VERSION = '1.0.0'
+
+      # Version of the states schema. Used to publish per partition simple aggregated metrics
+      # that can be used for schedules reporting
+      STATES_SCHEMA_VERSION = '1.0.0'
+
+      class << self
+        # Runs the `Proxy.call`
+        # @param kwargs [Hash] things requested by the proxy
+        # @return [Hash] message wrapped with the scheduled message envelope
+        def schedule(**kwargs)
+          Proxy.schedule(**kwargs)
+        end
+
+        # Generates a tombstone message to cancel given dispatch (if not yet happened)
+        # @param kwargs [Hash] things requested by the proxy
+        # @return [Hash] tombstone cancelling message
+        def cancel(**kwargs)
+          Proxy.cancel(**kwargs)
+        end
+
+        # Below are private APIs
+
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:scheduled_messages, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/processing/coordinator.rb

@@ -15,6 +15,13 @@ module Karafka
 
       attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      # This can be set directly on the listener because it can be triggered on first run without
+      # any messages
+      attr_accessor :eofed
+
+      # Last polled at time set based on the incoming last poll time
+      attr_accessor :last_polled_at
+
       def_delegators :@pause_tracker, :attempt, :paused?
 
       # @param topic [Karafka::Routing::Topic]
@@ -32,7 +39,9 @@ module Karafka
         @mutex = Mutex.new
         @marked = false
         @failure = false
+        @eofed = false
         @changed_at = monotonic_now
+        @last_polled_at = @changed_at
       end
 
       # Starts the coordinator for given consumption jobs
@@ -146,6 +155,11 @@ module Karafka
         @revoked
       end
 
+      # @return [Boolean] did we reach end of partition when polling data
+      def eofed?
+        @eofed
+      end
+
       # @return [Boolean] was the new seek offset assigned at least once. This is needed because
       #   by default we assign seek offset of a first message ever, however this is insufficient
       #   for DLQ in a scenario where the first message would be broken. We would never move

data/lib/karafka/processing/executor.rb

@@ -81,6 +81,13 @@ module Karafka
         consumer.on_before_consume
       end
 
+      # Runs the wrap/around execution context appropriate for a given action
+      # @param action [Symbol] action execution wrapped with our block
+      # @param block [Proc] execution context
+      def wrap(action, &block)
+        consumer.on_wrap(action, &block)
+      end
+
       # Runs consumer data processing against given batch and handles failures and errors.
       def consume
         # We run the consumer client logic...
@@ -104,6 +111,18 @@ module Karafka
         consumer.on_idle
       end
 
+      # Runs the code needed before eofed work is scheduled
+      def before_schedule_eofed
+        consumer.on_before_schedule_eofed
+      end
+
+      # Runs consumed eofed operation.
+      # This may run even when there were no messages received prior. This will however not
+      # run when eof is received together with messages as in such case `#consume` will run
+      def eofed
+        consumer.on_eofed
+      end
+
       # Runs code needed before revoked job is scheduled
       def before_schedule_revoked
         consumer.on_before_schedule_revoked if @consumer
@@ -162,13 +181,22 @@ module Karafka
           expansions.each { |expansion| consumer.singleton_class.include(expansion) }
 
           consumer.client = @client
-          consumer.producer = ::Karafka::App.producer
           consumer.coordinator = @coordinator
+          # We assign producer only when not available already. It may already be available if
+          # user redefined the `#producer` method for example. This can be useful for example when
+          # having a multi-cluster setup and using a totally custom producer
+          consumer.producer ||= ::Karafka::App.producer
           # Since we have some message-less flows (idle, etc), we initialize consumer with empty
           # messages set. In production we have persistent consumers, so this is not a performance
           # overhead as this will happen only once per consumer lifetime
           consumer.messages = empty_messages
 
+          # Run the post-initialization hook for users that need to run some actions when consumer
+          # is built and ready (all basic state and info).
+          # Users should **not** overwrite the `#initialize` because it won't have all the needed
+          # data assigned yet.
+          consumer.on_initialized
+
           consumer
         end
       end

data/lib/karafka/processing/jobs/base.rb

@@ -15,6 +15,11 @@ module Karafka
 
         attr_reader :executor
 
+        class << self
+          # @return [Symbol] Job matching appropriate action
+          attr_accessor :action
+        end
+
         # Creates a new job instance
         def initialize
           # All jobs are blocking by default and they can release the lock when blocking operations
@@ -23,6 +28,14 @@ module Karafka
           @status = :pending
         end
 
+        # Runs the wrap/around job hook within which the rest of the flow happens
+        # @param block [Proc] whole user related processing flow
+        # @note We inject the action name so user can decide whether to run custom logic on a
+        #   given action or not.
+        def wrap(&block)
+          executor.wrap(self.class.action, &block)
+        end
+
         # When redefined can run any code prior to the job being scheduled
         # @note This will run in the listener thread and not in the worker
         def before_schedule

data/lib/karafka/processing/jobs/consume.rb

@@ -9,6 +9,8 @@ module Karafka
         # @return [Array<Rdkafka::Consumer::Message>] array with messages
         attr_reader :messages
 
+        self.action = :consume
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch

data/lib/karafka/processing/jobs/eofed.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs the eofed operation when we receive eof without messages alongside.
+      class Eofed < Base
+        self.action = :eofed
+
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
+        # @return [Eofed]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs code prior to scheduling this eofed job
+        def before_schedule
+          executor.before_schedule_eofed
+        end
+
+        # Runs the eofed job via an executor.
+        def call
+          executor.eofed
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/idle.rb

@@ -6,6 +6,8 @@ module Karafka
       # Type of job that we may use to run some extra handling that happens without the user
       # related lifecycle event like consumption, revocation, etc.
       class Idle < Base
+        self.action = :idle
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job on an active consumer
         # @return [Shutdown]

data/lib/karafka/processing/jobs/revoked.rb

@@ -5,6 +5,8 @@ module Karafka
     module Jobs
       # Job that runs the revoked operation when we loose a partition on a consumer that lost it.
       class Revoked < Base
+        self.action = :revoked
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
         # @return [Revoked]
         def initialize(executor)