karafka 2.5.0.beta2 → 2.5.0.rc2

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -53,7 +53,7 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
-            # If this is set to true, it indicates unrecoverable error like fencing
+            # If this is set to a symbol, it indicates unrecoverable error like fencing
             # While fencing can be partial (for one of the SGs), we still should consider this
             # as an undesired state for the whole process because it halts processing in a
             # non-recoverable manner forever
@@ -116,7 +116,7 @@ module Karafka
             # We mark as unrecoverable only on certain errors that will not be fixed by retrying
             return unless UNRECOVERABLE_RDKAFKA_ERRORS.include?(error.code)
 
-            @unrecoverable = true
+            @unrecoverable = error.code
           end
 
           # Deregister the polling tracker for given listener
@@ -142,17 +142,29 @@ module Karafka
           # Did we exceed any of the ttls
           # @return [String] 204 string if ok, 500 otherwise
           def healthy?
-            time = monotonic_now
-
             return false if @unrecoverable
-            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+            return false if polling_ttl_exceeded?
+            return false if consuming_ttl_exceeded?
 
             true
           end
 
           private
 
+          # @return [Boolean] true if the consumer exceeded the polling ttl
+          def polling_ttl_exceeded?
+            time = monotonic_now
+
+            @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+          end
+
+          # @return [Boolean] true if the consumer exceeded the consuming ttl
+          def consuming_ttl_exceeded?
+            time = monotonic_now
+
+            @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+          end
+
           # Wraps the logic with a mutex
           # @param block [Proc] code we want to run in mutex
           def synchronize(&block)
@@ -191,6 +203,17 @@ module Karafka
               @consumptions.delete(thread_id)
             end
           end
+
+          # @return [Hash] response body status
+          def status_body
+            super.merge!(
+              errors: {
+                polling_ttl_exceeded: polling_ttl_exceeded?,
+                consumption_ttl_exceeded: consuming_ttl_exceeded?,
+                unrecoverable: @unrecoverable
+              }
+            )
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb

@@ -47,6 +47,15 @@ module Karafka
           def healthy?
             (monotonic_now - @controlling) < @controlling_ttl
           end
+
+          # @return [Hash] response body status
+          def status_body
+            super.merge!(
+              errors: {
+                controlling_ttl_exceeded: !healthy?
+              }
+            )
+          end
         end
       end
     end

data/lib/karafka/pro/encryption.rb

@@ -22,7 +22,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          Encryption::Contracts::Config.new.validate!(config.to_h)
+          Encryption::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           # Don't inject extra components if encryption is not active
           return unless config.encryption.active

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -22,6 +22,9 @@ module Karafka
           # @return [Hash]
           attr_reader :counts
 
+          # @return [String]
+          attr_reader :trace_id
+
           # Max errors we keep in memory.
           # We do not want to keep more because for DLQ-less this would cause memory-leaks.
           # We do however count per class for granular error counting
@@ -41,6 +44,7 @@ module Karafka
             @topic = topic
             @partition = partition
             @limit = limit
+            @trace_id = SecureRandom.uuid
           end
 
           # Clears all the errors
@@ -54,6 +58,7 @@ module Karafka
             @errors.shift if @errors.size >= @limit
             @errors << error
             @counts[error.class] += 1
+            @trace_id = SecureRandom.uuid
           end
 
           # @return [Boolean] is the error tracker empty

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -149,15 +149,16 @@ module Karafka
 
               dlq_message = {
                 topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
-                key: source_partition,
+                key: skippable_message.raw_key,
+                partition_key: source_partition,
                 payload: skippable_message.raw_payload,
                 headers: skippable_message.raw_headers.merge(
                   'source_topic' => topic.name,
                   'source_partition' => source_partition,
                   'source_offset' => skippable_message.offset.to_s,
                   'source_consumer_group' => topic.consumer_group.id,
-                  'source_key' => skippable_message.raw_key.to_s,
-                  'source_attempts' => attempt.to_s
+                  'source_attempts' => attempt.to_s,
+                  'source_trace_id' => errors_tracker.trace_id
                 )
               }
 

data/lib/karafka/pro/recurring_tasks.rb

@@ -29,7 +29,10 @@ module Karafka
           @schedule.instance_exec(&block)
 
           @schedule.each do |task|
-            Contracts::Task.new.validate!(task.to_h)
+            Contracts::Task.new.validate!(
+              task.to_h,
+              scope: ['recurring_tasks', task.id]
+            )
           end
 
           @schedule
@@ -59,7 +62,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+          RecurringTasks::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           # Published after task is successfully executed
           Karafka.monitor.notifications_bus.register_event('recurring_tasks.task.executed')

data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb

@@ -28,7 +28,8 @@ module Karafka
               # Validates that each node has at least one assignment.
               #
               # @param builder [Karafka::Routing::Builder]
-              def validate!(builder)
+              # @param scope [Array<String>]
+              def validate!(builder, scope: [])
                 nodes_setup = Hash.new do |h, node_id|
                   h[node_id] = { active: false, node_id: node_id }
                 end
@@ -49,7 +50,7 @@ module Karafka
                 end
 
                 nodes_setup.each_value do |details|
-                  super(details)
+                  super(details, scope: scope)
                 end
               end
 

data/lib/karafka/pro/routing/features/swarm.rb

@@ -17,7 +17,10 @@ module Karafka
             # @param config [Karafka::Core::Configurable::Node] app config
             def post_setup(config)
               config.monitor.subscribe('app.before_warmup') do
-                Contracts::Routing.new.validate!(config.internal.routing.builder)
+                Contracts::Routing.new.validate!(
+                  config.internal.routing.builder,
+                  scope: %w[swarm]
+                )
               end
             end
           end

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -12,13 +12,23 @@ module Karafka
           dispatcher_class: %i[scheduled_messages dispatcher_class]
         )
 
+        # In case there is an extremely high turnover of messages, EOF may never kick in,
+        # effectively not changing status from loading to loaded. We use the time consumer instance
+        # was created + a buffer time to detect such a case (loading + messages from the time it
+        # was already running) to switch the state despite no EOF
+        # This is in seconds
+        GRACE_PERIOD = 15
+
+        private_constant :GRACE_PERIOD
+
         # Prepares the initial state of all stateful components
         def initialized
           clear!
           # Max epoch is always moving forward with the time. Never backwards, hence we do not
           # reset it at all.
           @max_epoch = MaxEpoch.new
-          @state = State.new(nil)
+          @state = State.new
+          @reloads = 0
         end
 
         # Processes messages and runs dispatch (via tick) if needed
@@ -27,11 +37,25 @@ module Karafka
 
           messages.each do |message|
             SchemaValidator.call(message)
+
+            # We always track offsets of messages, even if they would be later on skipped or
+            # ignored for any reason. That way we have debug info that is useful once in a while.
+            @tracker.offsets(message)
+
             process_message(message)
           end
 
           @states_reporter.call
 
+          recent_timestamp = messages.last.timestamp.to_i
+          post_started_timestamp = @tracker.started_at + GRACE_PERIOD
+
+          # If we started getting messages that are beyond the current time, it means we have
+          # loaded enough to start scheduling. The upcoming messages are from the future looking
+          # from perspective of the current consumer start. We add a bit of grace period not to
+          # deal with edge cases
+          loaded! if @state.loading? && recent_timestamp > post_started_timestamp
+
           eofed if eofed?
 
           # Unless given day data is fully loaded we should not dispatch any notifications nor
@@ -55,11 +79,7 @@ module Karafka
           return if reload!
 
           # If end of the partition is reached, it always means all data is loaded
-          @state.loaded!
-
-          tags.add(:state, @state.to_s)
-
-          @states_reporter.call
+          loaded!
         end
 
         # Performs periodic operations when no new data is provided to the topic partition
@@ -90,6 +110,12 @@ module Karafka
           @states_reporter.call
         end
 
+        # Move the state to shutdown and publish immediately
+        def shutdown
+          @state.stopped!
+          @states_reporter.call!
+        end
+
         private
 
         # Takes each message and adds it to the daily accumulator if needed or performs other
@@ -104,7 +130,7 @@ module Karafka
             time = message.headers['schedule_target_epoch']
 
             # Do not track historical below today as those will be reflected in the daily buffer
-            @tracker.track(message) if time >= @today.starts_at
+            @tracker.future(message) if time >= @today.starts_at
 
             if time > @today.ends_at || time < @max_epoch.to_i
               # Clean the message immediately when not needed (won't be scheduled) to preserve
@@ -132,6 +158,7 @@ module Karafka
           # If this is a new assignment we always need to seek from beginning to load the data
           if @state.fresh?
             clear!
+            @reloads += 1
             seek(:earliest)
 
             return true
@@ -143,6 +170,7 @@ module Karafka
           # If day has ended we reload and start new day with new schedules
           if @today.ended?
             clear!
+            @reloads += 1
             seek(:earliest)
 
             return true
@@ -151,6 +179,13 @@ module Karafka
           false
         end
 
+        # Moves the state to loaded and publishes the state update
+        def loaded!
+          @state.loaded!
+          tags.add(:state, @state.to_s)
+          @states_reporter.call!
+        end
+
         # Resets all buffers and states so we can start a new day with a clean slate
         # We can fully recreate the dispatcher because any undispatched messages will be dispatched
         # with the new day dispatcher after it is reloaded.
@@ -158,11 +193,13 @@ module Karafka
           @daily_buffer = DailyBuffer.new
           @today = Day.new
           @tracker = Tracker.new
-          @state = State.new(false)
+          @state = State.new
+          @state.loading!
           @dispatcher = dispatcher_class.new(topic.name, partition)
           @states_reporter = Helpers::IntervalRunner.new do
             @tracker.today = @daily_buffer.size
             @tracker.state = @state.to_s
+            @tracker.reloads = @reloads
 
             @dispatcher.state(@tracker)
           end

data/lib/karafka/pro/scheduled_messages/dispatcher.rb

@@ -70,7 +70,8 @@ module Karafka
           config.producer.produce_async(
             topic: "#{@topic}#{config.states_postfix}",
             payload: @serializer.state(tracker),
-            key: 'state',
+            # We use the state as a key, so we always have one state transition data available
+            key: "#{tracker.state}_state",
             partition: @partition,
             headers: { 'zlib' => 'true' }
           )

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -60,7 +60,11 @@ module Karafka
             # We need to ensure that the message we want to proxy is fully legit. Otherwise, since
             # we envelope details like target topic, we could end up having incorrect data to
             # schedule
-            MSG_CONTRACT.validate!(message, WaterDrop::Errors::MessageInvalidError)
+            MSG_CONTRACT.validate!(
+              message,
+              WaterDrop::Errors::MessageInvalidError,
+              scope: %w[scheduled_messages message]
+            )
 
             headers = (message[:headers] || {}).merge(
               'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
@@ -166,9 +170,17 @@ module Karafka
           # complies with our requirements
           # @param proxy_message [Hash] our message envelope
           def validate!(proxy_message)
-            POST_CONTRACT.validate!(proxy_message)
+            POST_CONTRACT.validate!(
+              proxy_message,
+              scope: %w[scheduled_messages message]
+            )
+
             # After proxy specific validations we also ensure, that the final form is correct
-            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+            MSG_CONTRACT.validate!(
+              proxy_message,
+              WaterDrop::Errors::MessageInvalidError,
+              scope: %w[scheduled_messages message]
+            )
           end
         end
       end

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -16,10 +16,8 @@ module Karafka
         def state(tracker)
           data = {
             schema_version: ScheduledMessages::STATES_SCHEMA_VERSION,
-            dispatched_at: float_now,
-            state: tracker.state,
-            daily: tracker.daily
-          }
+            dispatched_at: float_now
+          }.merge(tracker.to_h)
 
           compress(
             serialize(data)

data/lib/karafka/pro/scheduled_messages/state.rb

@@ -15,38 +15,35 @@ module Karafka
       # - loaded - state in which we finished loading all the schedules and we can dispatch
       #   messages when the time comes and we can process real-time incoming schedules and
       #   changes to schedules as they appear in the stream.
+      # - shutdown - the states are no longer available as the consumer has shut down
       class State
-        # @param loaded [nil, false, true] is the state loaded or not yet. `nil` indicates, it is
-        #   a fresh, pre-seek state.
-        def initialize(loaded = nil)
-          @loaded = loaded
-        end
+        # Available states scheduling of messages may be in
+        STATES = %w[
+          fresh
+          loading
+          loaded
+          stopped
+        ].freeze
 
-        # @return [Boolean] are we in a fresh, pre-bootstrap state
-        def fresh?
-          @loaded.nil?
-        end
+        private_constant :STATES
 
-        # Marks the current state as fully loaded
-        def loaded!
-          @loaded = true
+        def initialize
+          @state = 'fresh'
         end
 
-        # @return [Boolean] are we in a loaded state
-        def loaded?
-          @loaded == true
+        STATES.each do |state|
+          define_method :"#{state}!" do
+            @state = state
+          end
+
+          define_method :"#{state}?" do
+            @state == state
+          end
         end
 
         # @return [String] current state string representation
         def to_s
-          case @loaded
-          when nil
-            'fresh'
-          when false
-            'loading'
-          when true
-            'loaded'
-          end
+          @state
         end
       end
     end

data/lib/karafka/pro/scheduled_messages/tracker.rb

@@ -10,25 +10,40 @@ module Karafka
       #
       # It provides accurate today dispatch taken from daily buffer and estimates for future days
       class Tracker
-        # @return [Hash<String, Integer>]
-        attr_reader :daily
-
         # @return [String] current state
         attr_accessor :state
 
+        attr_writer :reloads
+
+        # @return [Integer] time epoch when this tracker was started
+        attr_reader :started_at
+
         def initialize
           @daily = Hash.new { |h, k| h[k] = 0 }
-          @created_at = Time.now.to_i
+          @started_at = Time.now.to_i
+          @offsets = { low: -1, high: -1 }
+          @state = 'fresh'
+          @reloads = 0
         end
 
-        # Accurate (because coming from daily buffer) number of things to schedule
+        # Tracks offsets of visited messages
+        #
+        # @param message [Karafka::Messages::Message]
+        def offsets(message)
+          message_offset = message.offset
+
+          @offsets[:low] = message_offset if @offsets[:low].negative?
+          @offsets[:high] = message.offset
+        end
+
+        # Accurate (because coming from daily buffer) number of things to schedule daily
         #
         # @param sum [Integer]
         def today=(sum)
-          @daily[epoch_to_date(@created_at)] = sum
+          @daily[epoch_to_date(@started_at)] = sum
         end
 
-        # Tracks message dispatch
+        # Tracks future message dispatch
         #
         # It is only relevant for future days as for today we use accurate metrics from the daily
         # buffer
@@ -37,12 +52,23 @@ module Karafka
         #   tombstone message. Tombstone messages cancellations are not tracked because it would
         #   drastically increase complexity. For given day we use the accurate counter and for
         #   future days we use estimates.
-        def track(message)
+        def future(message)
           epoch = message.headers['schedule_target_epoch']
 
           @daily[epoch_to_date(epoch)] += 1
         end
 
+        # @return [Hash] hash with details that we want to expose
+        def to_h
+          {
+            state: @state,
+            offsets: @offsets,
+            daily: @daily,
+            started_at: @started_at,
+            reloads: @reloads
+          }.freeze
+        end
+
         private
 
         # @param epoch [Integer] epoch time

data/lib/karafka/pro/scheduled_messages.rb

@@ -51,7 +51,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+          ScheduledMessages::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
         end
 
         # Basically since we may have custom producers configured that are not the same as the

data/lib/karafka/routing/builder.rb

@@ -50,15 +50,24 @@ module Karafka
 
           # Ensures high-level routing details consistency
           # Contains checks that require knowledge about all the consumer groups to operate
-          Contracts::Routing.new.validate!(map(&:to_h))
+          Contracts::Routing.new.validate!(
+            map(&:to_h),
+            scope: %w[routes]
+          )
 
           each do |consumer_group|
             # Validate consumer group settings
-            Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+            Contracts::ConsumerGroup.new.validate!(
+              consumer_group.to_h,
+              scope: ['routes', consumer_group.name]
+            )
 
             # and then its topics settings
             consumer_group.topics.each do |topic|
-              Contracts::Topic.new.validate!(topic.to_h)
+              Contracts::Topic.new.validate!(
+                topic.to_h,
+                scope: ['routes', consumer_group.name, topic.name]
+              )
             end
 
             # Initialize subscription groups after all the routing is done

data/lib/karafka/routing/features/base/expander.rb

@@ -38,13 +38,19 @@ module Karafka
 
                 each do |consumer_group|
                   if scope::Contracts.const_defined?('ConsumerGroup', false)
-                    scope::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+                    scope::Contracts::ConsumerGroup.new.validate!(
+                      consumer_group.to_h,
+                      scope: ['routes', consumer_group.name]
+                    )
                   end
 
                   next unless scope::Contracts.const_defined?('Topic', false)
 
                   consumer_group.topics.each do |topic|
-                    scope::Contracts::Topic.new.validate!(topic.to_h)
+                    scope::Contracts::Topic.new.validate!(
+                      topic.to_h,
+                      scope: ['routes', consumer_group.name, topic.name]
+                    )
                   end
                 end
 

data/lib/karafka/server.rb

@@ -51,7 +51,10 @@ module Karafka
         # embedded
         # We cannot validate this during the start because config needs to be populated and routes
         # need to be defined.
-        cli_contract.validate!(activity_manager.to_h)
+        cli_contract.validate!(
+          activity_manager.to_h,
+          scope: %w[cli]
+        )
 
         # We clear as we do not want parent handlers in case of working from fork
         process.clear

data/lib/karafka/setup/config.rb

@@ -131,11 +131,20 @@ module Karafka
         # option max_wait_time [Integer] We wait only for this amount of time before raising error
         # as we intercept this error and retry after checking that the operation was finished or
         # failed using external factor.
-        setting :max_wait_time, default: 1_000
+        #
+        # For async this will finish immediately but for sync operations this will wait and we
+        # will get a confirmation. 60 seconds is ok for both cases as for async, the re-wait will
+        # kick in
+        setting :max_wait_time, default: 60 * 1_000
+
+        # How long should we wait on admin operation retrying before giving up and raising an
+        # error that result is not visible
+        setting :max_retries_duration, default: 60_000
 
-        # How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
-        # up on pending operations
-        setting :max_attempts, default: 60
+        # In case of fast-finished async work, this `retry_backoff` help us not re-query Kafka
+        # too fast after previous call to check the async operation results. Basically prevents
+        # us from spamming metadata requests to Kafka in a loop
+        setting :retry_backoff, default: 500
 
         # option poll_timeout [Integer] time in ms
         # How long should a poll wait before yielding on no results (rdkafka-ruby setting)
@@ -352,7 +361,10 @@ module Karafka
 
           configure(&block)
 
-          Contracts::Config.new.validate!(config.to_h)
+          Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           configure_components
 

data/lib/karafka/swarm/supervisor.rb

@@ -42,7 +42,10 @@ module Karafka
       # Creates needed number of forks, installs signals and starts supervision
       def run
         # Validate the CLI provided options the same way as we do for the regular server
-        cli_contract.validate!(activity_manager.to_h)
+        cli_contract.validate!(
+          activity_manager.to_h,
+          scope: %w[swarm cli]
+        )
 
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
@@ -154,7 +157,7 @@ module Karafka
         # Run forceful kill
         manager.terminate
         # And wait until linux kills them
-        # This prevents us from existing forcefully with any dead child process still existing
+        # This prevents us from exiting forcefully with any dead child process still existing
         # Since we have sent the `KILL` signal, it must die, so we can wait until all dead
         sleep(supervision_sleep) until manager.stopped?
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.0.beta2'
+  VERSION = '2.5.0.rc2'
 end