karafka 2.5.0.beta2 → 2.5.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbcaf396d7f2eff35ec3b59c96ffe7dd880f1f07294aa28bb623e95ce328e3e9
-  data.tar.gz: 410b037a79abbbc82fbd4b540dcb26378dca396e408f68280325e354ae2e275b
+  metadata.gz: 2a66089d998c0dabb1070e4e8f1895a068e8f2aa8e752fb38ef9da1633b9704d
+  data.tar.gz: 188ea36894e0a32168303654510ef2e072d19f8ad39e5f0155547b6c96dbfdb2
 SHA512:
-  metadata.gz: dcd79f3bda653b74d95938d440ccfe93d8d2cdc214e02de31d9ad56f738b554699bd1991d83dd1c9c7090e1c3a63669f8837677fef77efc22df4804bc16f25a0
-  data.tar.gz: dbca53f433a0e13ec7582f9ee1c6da15bf38678a1019a3a36e361e7bbbae2c5e5777bf44e61330692b5f38c4fa011dd9e3950027925767908f617df02e1e8219
+  metadata.gz: d3ee8f86dd3b26dea69f9e03972ac3aced8b76d8156a9c359cb2b50114f3156306281a7f147935fa4d566e2d398f0a343205255a96f9560528b9dc2d21ca166c
+  data.tar.gz: 843e553470b78b107080df06ae7f7bd716d5d08363fb0d381709522aed397bb5da6d46c0d1aee06a72c6ba063b112118a669ff3f079fef4c53f672cf08ed1ee0

data/.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: ci
+name: CI
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
+        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
         with:
           ruby-version: 3.4
           bundler-cache: true
@@ -118,7 +118,7 @@ jobs:
         run: rm -f Gemfile.lock
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
+        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
@@ -164,7 +164,7 @@ jobs:
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
+        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
         with:
           # Do not use cache here as we run bundle install also later in some of the integration
           # tests and we need to be able to run it without cache
@@ -228,7 +228,7 @@ jobs:
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
+        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler: 'latest'

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@bb0f760b6c925183520ee0bcc9c4a432a7c8c3c6 # v1.241.0
+        uses: ruby/setup-ruby@13e7a03dc3ac6c3798f4570bfead2aed4d96abfb # v1.244.0
         with:
           bundler-cache: false
 
@@ -32,5 +32,4 @@ jobs:
         run: |
           bundle install --jobs 4 --retry 3
 
-      # Release
-      - uses: rubygems/release-gem@9e85cb11501bebc2ae661c1500176316d3987059 # v1
+      - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1

data/CHANGELOG.md

@@ -1,6 +1,8 @@
 # Karafka Framework Changelog
 
 ## 2.5.0 (Unreleased)
+- **[Breaking]** Change how consistency of DLQ dispatches works in Pro (`partition_key` vs. direct partition id mapping).
+- **[Breaking]** Remove the headers `source_key` from the Pro DLQ dispatched messages as the original key is now fully preserved.
 - **[Breaking]** Use DLQ and Piping prefix `source_` instead of `original_` to align with naming convention of Kafka Streams and Apache Flink for future usage.
 - **[Breaking]** Rename scheduled jobs topics names in their config (Pro).
 - **[Feature]** Parallel Segments for concurrent processing of the same partition with more than partition count of processes (Pro).
@@ -35,12 +37,17 @@
 - [Enhancement] Execute the help CLI command when no command provided (similar to Rails) to improve DX.
 - [Enhancement] Remove backtrace from the CLI error for incorrect commands (similar to Rails) to improve DX.
 - [Enhancement] Provide `karafka topics help` sub-help due to nesting of Declarative Topics actions.
+- [Enhancement] Use independent keys for different states of reporting in scheduled messages.
+- [Enhancement] Enrich scheduled messages state reporter with debug data.
+- [Enhancement] Introduce a new state called `stopped` to the scheduled messages.
+- [Enhancement] Do not overwrite the `key` in the Pro DLQ dispatched messages for routing reasons.
+- [Enhancement] Introduce `errors_tracker.trace_id` for distributed error details correlation with the Web UI. 
 - [Refactor] Introduce a `bin/verify_kafka_warnings` script to clean Kafka from temporary test-suite topics.
 - [Refactor] Introduce a `bin/verify_topics_naming` script to ensure proper test topics naming convention.
 - [Refactor] Make sure all temporary topics have a `it-` prefix in their name.
 - [Refactor] Improve CI specs parallelization.
 - [Maintenance] Lower the `Karafka::Admin` `poll_timeout` to 50 ms to improve responsiveness of admin operations.
-- [Maintenance] Require `karafka-rdkafka` `>=` `0.19.2` due to usage of `#rd_kafka_global_init`, KIP-82 and the new producer caching engine.
+- [Maintenance] Require `karafka-rdkafka` `>=` `0.19.5` due to usage of `#rd_kafka_global_init`, KIP-82, new producer caching engine and improvements to the `partition_key` assignments.
 - [Maintenance] Add Deimos routing patch into integration suite not to break it in the future.
 - [Maintenance] Remove Rails `7.0` specs due to upcoming EOL.
 - [Fix] Fix Recurring Tasks and Scheduled Messages not working with Swarm (using closed producer).
@@ -58,6 +65,7 @@
 - [Fix] `karafka` cannot be required without Bundler.
 - [Fix] Scheduled Messages re-seek moves to `latest` on inheritance of initial offset when `0` offset is compacted.
 - [Fix] Seek to `:latest` without `topic_partition_position` (-1) will not seek at all.
+- [Fix] Extremely high turn over of scheduled messages can cause them not to reach EOF/Loaded state.
 - [Change] Move to trusted-publishers and remove signing since no longer needed.
 
 ## 2.4.18 (2025-04-09)

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.5.0.beta2)
+    karafka (2.5.0.rc1)
       base64 (~> 0.2)
       karafka-core (>= 2.5.0, < 2.6.0)
-      karafka-rdkafka (>= 0.19.2)
+      karafka-rdkafka (>= 0.19.5)
       waterdrop (>= 2.8.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -28,7 +28,7 @@ GEM
       tzinfo (~> 2.0, >= 2.0.5)
       uri (>= 0.13.1)
     base64 (0.2.0)
-    benchmark (0.4.0)
+    benchmark (0.4.1)
     bigdecimal (3.1.9)
     byebug (12.0.0)
     concurrent-ruby (1.3.5)
@@ -39,7 +39,7 @@ GEM
     erubi (1.13.1)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.5.1)
+    factory_bot (6.5.2)
       activesupport (>= 6.1.0)
     ffi (1.17.2)
     ffi (1.17.2-aarch64-linux-gnu)
@@ -62,11 +62,11 @@ GEM
     karafka-core (2.5.1)
       karafka-rdkafka (>= 0.19.2, < 0.21.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.19.4)
+    karafka-rdkafka (0.19.5)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.5.0)
+    karafka-testing (2.5.1)
       karafka (>= 2.5.0.beta1, < 2.6.0)
       waterdrop (>= 2.8.0)
     karafka-web (0.11.0.beta3)
@@ -81,22 +81,22 @@ GEM
     ostruct (0.6.1)
     raabro (1.4.0)
     rack (3.1.15)
-    rake (13.2.1)
+    rake (13.3.0)
     roda (3.92.0)
       rack
-    rspec (3.13.0)
+    rspec (3.13.1)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.3)
+    rspec-core (3.13.4)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.4)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.4)
+    rspec-mocks (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.3)
+    rspec-support (3.13.4)
     securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)

data/karafka.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.5.0', '< 2.6.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.19.2'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.19.5'
   spec.add_dependency 'waterdrop', '>= 2.8.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/connection/client.rb

@@ -427,6 +427,15 @@ module Karafka
         @wrapped_kafka.committed(tpl)
       end
 
+      # Reads watermark offsets for given topic
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @return [Array<Integer, Integer>] watermark offsets (low, high)
+      def query_watermark_offsets(topic, partition)
+        @wrapped_kafka.query_watermark_offsets(topic, partition)
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition

data/lib/karafka/connection/proxy.rb

@@ -44,7 +44,7 @@ module Karafka
       # clusters can handle our requests.
       #
       # @param topic [String] topic name
-      # @param partition [Partition]
+      # @param partition [Integer] partition number
       # @return [Array<Integer, Integer>] watermark offsets
       def query_watermark_offsets(topic, partition)
         l_config = @config.query_watermark_offsets

data/lib/karafka/helpers/interval_runner.rb

@@ -30,6 +30,14 @@ module Karafka
         @block.call
       end
 
+      # Runs the requested code bypassing any time frequencies
+      # Useful when we have certain actions that usually need to run periodically but in some
+      # cases need to run asap
+      def call!
+        @last_called_at = monotonic_now
+        @block.call
+      end
+
       # Resets the runner, so next `#call` will run the underlying code
       def reset
         @last_called_at = monotonic_now - @interval

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -22,6 +22,9 @@ module Karafka
           # @return [Hash]
           attr_reader :counts
 
+          # @return [String]
+          attr_reader :trace_id
+
           # Max errors we keep in memory.
           # We do not want to keep more because for DLQ-less this would cause memory-leaks.
           # We do however count per class for granular error counting
@@ -41,6 +44,7 @@ module Karafka
             @topic = topic
             @partition = partition
             @limit = limit
+            @trace_id = SecureRandom.uuid
           end
 
           # Clears all the errors
@@ -54,6 +58,7 @@ module Karafka
             @errors.shift if @errors.size >= @limit
             @errors << error
             @counts[error.class] += 1
+            @trace_id = SecureRandom.uuid
           end
 
           # @return [Boolean] is the error tracker empty

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -149,15 +149,16 @@ module Karafka
 
               dlq_message = {
                 topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
-                key: source_partition,
+                key: skippable_message.raw_key,
+                partition_key: source_partition,
                 payload: skippable_message.raw_payload,
                 headers: skippable_message.raw_headers.merge(
                   'source_topic' => topic.name,
                   'source_partition' => source_partition,
                   'source_offset' => skippable_message.offset.to_s,
                   'source_consumer_group' => topic.consumer_group.id,
-                  'source_key' => skippable_message.raw_key.to_s,
-                  'source_attempts' => attempt.to_s
+                  'source_attempts' => attempt.to_s,
+                  'source_trace_id' => errors_tracker.trace_id
                 )
               }
 

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -12,13 +12,23 @@ module Karafka
           dispatcher_class: %i[scheduled_messages dispatcher_class]
         )
 
+        # In case there is an extremely high turnover of messages, EOF may never kick in,
+        # effectively not changing status from loading to loaded. We use the time consumer instance
+        # was created + a buffer time to detect such a case (loading + messages from the time it
+        # was already running) to switch the state despite no EOF
+        # This is in seconds
+        GRACE_PERIOD = 15
+
+        private_constant :GRACE_PERIOD
+
         # Prepares the initial state of all stateful components
         def initialized
           clear!
           # Max epoch is always moving forward with the time. Never backwards, hence we do not
           # reset it at all.
           @max_epoch = MaxEpoch.new
-          @state = State.new(nil)
+          @state = State.new
+          @reloads = 0
         end
 
         # Processes messages and runs dispatch (via tick) if needed
@@ -27,11 +37,25 @@ module Karafka
 
           messages.each do |message|
             SchemaValidator.call(message)
+
+            # We always track offsets of messages, even if they would be later on skipped or
+            # ignored for any reason. That way we have debug info that is useful once in a while.
+            @tracker.offsets(message)
+
             process_message(message)
           end
 
           @states_reporter.call
 
+          recent_timestamp = messages.last.timestamp.to_i
+          post_started_timestamp = @tracker.started_at + GRACE_PERIOD
+
+          # If we started getting messages that are beyond the current time, it means we have
+          # loaded enough to start scheduling. The upcoming messages are from the future looking
+          # from perspective of the current consumer start. We add a bit of grace period not to
+          # deal with edge cases
+          loaded! if @state.loading? && recent_timestamp > post_started_timestamp
+
           eofed if eofed?
 
           # Unless given day data is fully loaded we should not dispatch any notifications nor
@@ -55,11 +79,7 @@ module Karafka
           return if reload!
 
           # If end of the partition is reached, it always means all data is loaded
-          @state.loaded!
-
-          tags.add(:state, @state.to_s)
-
-          @states_reporter.call
+          loaded!
         end
 
         # Performs periodic operations when no new data is provided to the topic partition
@@ -90,6 +110,12 @@ module Karafka
           @states_reporter.call
         end
 
+        # Move the state to shutdown and publish immediately
+        def shutdown
+          @state.stopped!
+          @states_reporter.call!
+        end
+
         private
 
         # Takes each message and adds it to the daily accumulator if needed or performs other
@@ -104,7 +130,7 @@ module Karafka
             time = message.headers['schedule_target_epoch']
 
             # Do not track historical below today as those will be reflected in the daily buffer
-            @tracker.track(message) if time >= @today.starts_at
+            @tracker.future(message) if time >= @today.starts_at
 
             if time > @today.ends_at || time < @max_epoch.to_i
               # Clean the message immediately when not needed (won't be scheduled) to preserve
@@ -132,6 +158,7 @@ module Karafka
           # If this is a new assignment we always need to seek from beginning to load the data
           if @state.fresh?
             clear!
+            @reloads += 1
             seek(:earliest)
 
             return true
@@ -143,6 +170,7 @@ module Karafka
           # If day has ended we reload and start new day with new schedules
           if @today.ended?
             clear!
+            @reloads += 1
             seek(:earliest)
 
             return true
@@ -151,6 +179,13 @@ module Karafka
           false
         end
 
+        # Moves the state to loaded and publishes the state update
+        def loaded!
+          @state.loaded!
+          tags.add(:state, @state.to_s)
+          @states_reporter.call!
+        end
+
         # Resets all buffers and states so we can start a new day with a clean slate
         # We can fully recreate the dispatcher because any undispatched messages will be dispatched
         # with the new day dispatcher after it is reloaded.
@@ -158,11 +193,13 @@ module Karafka
           @daily_buffer = DailyBuffer.new
           @today = Day.new
           @tracker = Tracker.new
-          @state = State.new(false)
+          @state = State.new
+          @state.loading!
           @dispatcher = dispatcher_class.new(topic.name, partition)
           @states_reporter = Helpers::IntervalRunner.new do
             @tracker.today = @daily_buffer.size
             @tracker.state = @state.to_s
+            @tracker.reloads = @reloads
 
             @dispatcher.state(@tracker)
           end

data/lib/karafka/pro/scheduled_messages/dispatcher.rb

@@ -70,7 +70,8 @@ module Karafka
           config.producer.produce_async(
             topic: "#{@topic}#{config.states_postfix}",
             payload: @serializer.state(tracker),
-            key: 'state',
+            # We use the state as a key, so we always have one state transition data available
+            key: "#{tracker.state}_state",
             partition: @partition,
             headers: { 'zlib' => 'true' }
           )

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -16,10 +16,8 @@ module Karafka
         def state(tracker)
           data = {
             schema_version: ScheduledMessages::STATES_SCHEMA_VERSION,
-            dispatched_at: float_now,
-            state: tracker.state,
-            daily: tracker.daily
-          }
+            dispatched_at: float_now
+          }.merge(tracker.to_h)
 
           compress(
             serialize(data)

data/lib/karafka/pro/scheduled_messages/state.rb

@@ -15,38 +15,35 @@ module Karafka
       # - loaded - state in which we finished loading all the schedules and we can dispatch
       #   messages when the time comes and we can process real-time incoming schedules and
       #   changes to schedules as they appear in the stream.
+      # - shutdown - the states are no longer available as the consumer has shut down
       class State
-        # @param loaded [nil, false, true] is the state loaded or not yet. `nil` indicates, it is
-        #   a fresh, pre-seek state.
-        def initialize(loaded = nil)
-          @loaded = loaded
-        end
+        # Available states scheduling of messages may be in
+        STATES = %w[
+          fresh
+          loading
+          loaded
+          stopped
+        ].freeze
 
-        # @return [Boolean] are we in a fresh, pre-bootstrap state
-        def fresh?
-          @loaded.nil?
-        end
+        private_constant :STATES
 
-        # Marks the current state as fully loaded
-        def loaded!
-          @loaded = true
+        def initialize
+          @state = 'fresh'
         end
 
-        # @return [Boolean] are we in a loaded state
-        def loaded?
-          @loaded == true
+        STATES.each do |state|
+          define_method :"#{state}!" do
+            @state = state
+          end
+
+          define_method :"#{state}?" do
+            @state == state
+          end
         end
 
         # @return [String] current state string representation
         def to_s
-          case @loaded
-          when nil
-            'fresh'
-          when false
-            'loading'
-          when true
-            'loaded'
-          end
+          @state
         end
       end
     end

data/lib/karafka/pro/scheduled_messages/tracker.rb

@@ -10,25 +10,40 @@ module Karafka
       #
       # It provides accurate today dispatch taken from daily buffer and estimates for future days
       class Tracker
-        # @return [Hash<String, Integer>]
-        attr_reader :daily
-
         # @return [String] current state
         attr_accessor :state
 
+        attr_writer :reloads
+
+        # @return [Integer] time epoch when this tracker was started
+        attr_reader :started_at
+
         def initialize
           @daily = Hash.new { |h, k| h[k] = 0 }
-          @created_at = Time.now.to_i
+          @started_at = Time.now.to_i
+          @offsets = { low: -1, high: -1 }
+          @state = 'fresh'
+          @reloads = 0
         end
 
-        # Accurate (because coming from daily buffer) number of things to schedule
+        # Tracks offsets of visited messages
+        #
+        # @param message [Karafka::Messages::Message]
+        def offsets(message)
+          message_offset = message.offset
+
+          @offsets[:low] = message_offset if @offsets[:low].negative?
+          @offsets[:high] = message.offset
+        end
+
+        # Accurate (because coming from daily buffer) number of things to schedule daily
         #
         # @param sum [Integer]
         def today=(sum)
-          @daily[epoch_to_date(@created_at)] = sum
+          @daily[epoch_to_date(@started_at)] = sum
         end
 
-        # Tracks message dispatch
+        # Tracks future message dispatch
         #
         # It is only relevant for future days as for today we use accurate metrics from the daily
         # buffer
@@ -37,12 +52,23 @@ module Karafka
         #   tombstone message. Tombstone messages cancellations are not tracked because it would
         #   drastically increase complexity. For given day we use the accurate counter and for
         #   future days we use estimates.
-        def track(message)
+        def future(message)
           epoch = message.headers['schedule_target_epoch']
 
           @daily[epoch_to_date(epoch)] += 1
         end
 
+        # @return [Hash] hash with details that we want to expose
+        def to_h
+          {
+            state: @state,
+            offsets: @offsets,
+            daily: @daily,
+            started_at: @started_at,
+            reloads: @reloads
+          }.freeze
+        end
+
         private
 
         # @param epoch [Integer] epoch time

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.0.beta2'
+  VERSION = '2.5.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.0.beta2
+  version: 2.5.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -49,14 +49,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.19.2
+        version: 0.19.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.19.2
+        version: 0.19.5
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
@@ -146,10 +146,11 @@ files:
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
 - docker-compose.yml
-- examples/payloads/json/enrollment_event.json
-- examples/payloads/json/ingestion_event.json
-- examples/payloads/json/transaction_event.json
-- examples/payloads/json/user_event.json
+- examples/payloads/avro/.gitkeep
+- examples/payloads/json/sample_set_01/enrollment_event.json
+- examples/payloads/json/sample_set_01/ingestion_event.json
+- examples/payloads/json/sample_set_01/transaction_event.json
+- examples/payloads/json/sample_set_01/user_event.json
 - karafka.gemspec
 - lib/active_job/karafka.rb
 - lib/active_job/queue_adapters/karafka_adapter.rb