karafka 2.0.15 → 2.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79c991a988c67b681c8bdf16d06d61b3781be5bbc070d23a1258ea0152058d41
-  data.tar.gz: 478d2155761e7fa4bb6b766a5fd598f3524b5810edddc36229185bb9ce396c3c
+  metadata.gz: 8cc0ac8b318d7c40e8974e2e26753c68cc5f611bbcd0e4fbea94e3eb275002dd
+  data.tar.gz: db638edaecc662ac8a0f7d627f4bcae6e1e1e83500a49c4dfa4a4f32ee8f78c9
 SHA512:
-  metadata.gz: d9a3ba7e201947f0ced657eb61aa8f4af348f70a85167a9194f34039d070248551aa3a09d270930ccac399d774fc89305a4f2bc36da715066cd789997fbb40d9
-  data.tar.gz: b5cf1015aa851c7ab78fa89e2b2317085c18f24c8b5a03f5b82c8c3c7ac2ef95fe6a1888defc97f62492c107c2b0b2cd1b72bbe5811fb99dd92e39e93f912608
+  metadata.gz: 6875e7e152b4150f397f3830c828412e0477409483c81e714525161e5263cff686c0cb9b46a6c11cb5b914dc51087e4bac95d5dcdc0e25c7233825b363ed7e6c
+  data.tar.gz: 1f35d410350dc3fbe97462e979f3df832484df1f1f5119e16fe2c28210d01a05e1d3e086c73e1a3b70ba3c6cae9a25dd50e0907ca454b68c9166efcabd22f972

data/.github/workflows/ci.yml

@@ -81,7 +81,7 @@ jobs:
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
-        run: bundle exec rspec
+        run: bin/rspecs
 
   integrations:
     runs-on: ubuntu-latest

data/.rspec

@@ -1,3 +1,5 @@
 --require spec_helper
 # Integration specs run with their one framework, not via RSpec
 --exclude-pattern "spec/integrations/**/*_spec.rb"
+# Do not run pro at the same time as default specs as it would cause state conflicts
+--tag ~type:pro

data/CHANGELOG.md

@@ -1,5 +1,83 @@
 # Karafka framework changelog
 
+## 2.0.16 (2022-11-09)
+- **[Breaking]** Disable the root `manual_offset_management` setting and require it to be configured per topic. This is part of "topic features" configuration extraction for better code organization.
+- **[Feature]** Introduce **Dead Letter Queue** feature and Pro **Enhanced Dead Letter Queue** feature
+- [Improvement] Align attributes available in the instrumentation bus for listener related events.
+- [Improvement] Include consumer group id in consumption related events (#1093)
+- [Improvement] Delegate pro components loading to Zeitwerk
+- [Improvement] Include `Datadog::LoggerListener` for tracking logger data with DataDog (@bruno-b-martins)
+- [Improvement] Include `seek_offset` in the `consumer.consume.error` event payload (#1113)
+- [Refactor] Remove unused logger listener event handler.
+- [Refactor] Internal refactoring of routing validations flow.
+- [Refactor] Reorganize how routing related features are represented internally to simplify features management.
+- [Refactor] Extract supported features combinations processing flow into separate strategies.
+- [Refactor] Auto-create topics in the integration specs based on the defined routing
+- [Refactor] Auto-inject Pro components via composition instead of requiring to use `Karafka::Pro::BaseConsumer` (#1116)
+- [Fix] Fix a case where routing tags would not be injected when given routing definition would not be used with a block
+- [Fix] Fix a case where using `#active_job_topic` without extra block options would cause `manual_offset_management` to stay false.
+- [Fix] Fix a case when upon Pro ActiveJob usage with Virtual Partitions, correct offset would not be stored
+- [Fix] Fix a case where upon Virtual Partitions usage, same underlying real partition would be resumed several times.
+- [Fix] Fix LRJ enqueuing pause increases the coordinator counter (#115)
+- [Fix] Release `ActiveRecord` connection to the pool after the work in non-dev envs (#1130)
+- [Fix] Fix a case where post-initialization shutdown would not initiate shutdown procedures.
+- [Fix] Prevent Karafka from committing offsets twice upon shutdown.
+- [Fix] Fix for a case where fast consecutive stop signaling could hang the stopping listeners.
+- [Specs] Split specs into regular and pro to simplify how resources are loaded
+- [Specs] Add specs to ensure, that all the Pro components have a proper per-file license (#1099)
+
+
+### Upgrade notes
+
+1. Remove the `manual_offset_management` setting from the main config if you use it:
+
+```ruby
+class KarafkaApp < Karafka::App
+  setup do |config|
+    # ...
+
+    # This line needs to be removed:
+    config.manual_offset_management = true
+  end
+end
+```
+
+2. Set the `manual_offset_management` feature flag per each topic where you want to use it in the routing. Don't set it for topics where you want the default offset management strategy to be used.
+
+```ruby
+class KarafkaApp < Karafka::App
+  routes.draw do
+    consumer_group :group_name do
+      topic :example do
+        consumer ExampleConsumer
+        manual_offset_management true
+      end
+
+      topic :example2 do
+        consumer ExampleConsumer2
+        manual_offset_management true
+      end
+    end
+  end
+end
+```
+
+3. If you were using code to restart dead connections similar to this:
+
+```ruby
+class ActiveRecordConnectionsCleaner
+  def on_error_occurred(event)
+    return unless event[:error].is_a?(ActiveRecord::StatementInvalid)
+
+    ::ActiveRecord::Base.clear_active_connections!
+  end
+end
+
+Karafka.monitor.subscribe(ActiveRecordConnectionsCleaner.new)
+```
+
+It **should** be removed. This code is **no longer needed**.
+
 ## 2.0.15 (2022-10-20)
 - Sanitize admin config prior to any admin action.
 - Make messages partitioner outcome for virtual partitions consistently distributed in regards to concurrency.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.15)
+    karafka (2.0.16)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
@@ -39,19 +39,19 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -65,7 +65,7 @@ GEM
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.1)
+    zeitwerk (2.6.6)
 
 PLATFORMS
   x86_64-linux
@@ -79,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.22
+   2.3.24

data/LICENSE

@@ -12,6 +12,6 @@ and their usage requires commercial license agreement.
 By sending a pull request to the pro components, you are agreeing to transfer the copyright of your
 code to Maciej Mensfeld.
 
-You can find the commercial license in LICENSE-COM.
+You can find the commercial license in LICENSE-COMM.
 
 Please see https://karafka.io for purchasing options.

data/README.md

@@ -9,8 +9,9 @@
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
 - Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
-- Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
+- Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
+- Has a seamless [Dead Letter Queue](karafka.io/docs/Dead-Letter-Queue/) functionality built-in
 - Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
 - Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.

data/bin/integrations

@@ -21,9 +21,10 @@ require 'etc'
 ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
 
 # How many child processes with integration specs do we want to run in parallel
-# When the value is high, there's a problem with thread allocation on Github CI, tht is why
+# When the value is high, there's a problem with thread allocation on Github CI, that is why
 # we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
-# of CPU
+# of CPU. Locally we also cannot go beyond certain limit due to how often and how many topics we
+# create in Kafka. With an overloaded system, we start getting timeouts.
 CONCURRENCY = ENV.key?('CI') ? 4 : Etc.nprocessors * 2
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it

data/bin/rspecs

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+SPECS_TYPE=regular bundle exec rspec --tag ~type:pro
+SPECS_TYPE=pro bundle exec rspec --tag type:pro

data/config/errors.yml

@@ -27,15 +27,20 @@ en:
       missing: needs to be present
       consumer_groups_inclusion: Unknown consumer group
 
-    consumer_group_topic:
+    topic:
       missing: needs to be present
       name_format: 'needs to be a string with a Kafka accepted format'
       deserializer_format: needs to be present
-      manual_offset_management_format: needs to be either true or false
       consumer_format: needs to be present
       id_format: 'needs to be a string with a Kafka accepted format'
       initial_offset_format: needs to be either earliest or latest
       subscription_group_format: must be nil or a non-empty string
+      manual_offset_management.active_format: needs to be either true or false
+      consumer_active_job_missing: ActiveJob needs to be available
+      manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
+      dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
+      dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
+      dead_letter_queue.active_format: needs to be either true or false
 
     consumer_group:
       missing: needs to be present
@@ -53,8 +58,9 @@ en:
       missing: needs to be present
       id_format: needs to be a String
 
-    pro_consumer_group_topic:
-      consumer_format: needs to inherit from Karafka::Pro::BaseConsumer and not Karafka::Consumer
+    pro_topic:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
       virtual_partitions.max_partitions_format: needs to be equl or more than 1
       manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+      long_running_job.active_format: needs to be either true or false
+      dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions

data/lib/active_job/karafka.rb

@@ -10,12 +10,6 @@ begin
     module Karafka
     end
   end
-
-  # We extend routing builder by adding a simple wrapper for easier jobs topics defining
-  # This needs to be extended here as it is going to be used in karafka routes, hence doing that in
-  # the railtie initializer would be too late
-  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::Routing::Extensions
-  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::Routing::Extensions
 rescue LoadError
   # We extend ActiveJob stuff in the railtie
 end

data/lib/karafka/active_job/consumer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for all the ActiveJob related things from within Karafka
   module ActiveJob
     # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
     # It marks the offset after each message, so we make sure, none of the jobs is executed twice

data/lib/karafka/admin.rb

@@ -21,7 +21,7 @@ module Karafka
         with_admin do |admin|
           admin.create_topic(name, partitions, replication_factor, topic_config)
 
-          sleep(0.1) until topics_names.include?(name)
+          sleep(0.2) until topics_names.include?(name)
         end
       end
 
@@ -32,7 +32,7 @@ module Karafka
         with_admin do |admin|
           admin.delete_topic(name)
 
-          sleep(0.1) while topics_names.include?(name)
+          sleep(0.2) while topics_names.include?(name)
         end
       end
 

data/lib/karafka/base_consumer.rb

@@ -21,7 +21,16 @@ module Karafka
     # @note This should not be used by the end users as it is part of the lifecycle of things and
     #   not as a part of the public api. This should not perform any extensive operations as it is
     #   blocking and running in the listener thread.
-    def on_before_enqueue; end
+    def on_before_enqueue
+      handle_before_enqueue
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_enqueue.error'
+      )
+    end
 
     # Can be used to run preparation code in the worker
     #
@@ -32,6 +41,17 @@ module Karafka
     def on_before_consume
       messages.metadata.processed_at = Time.now
       messages.metadata.freeze
+
+      # We run this after the full metadata setup, so we can use all the messages information
+      # if needed
+      handle_before_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.before_consume.error'
+      )
     end
 
     # Executes the default consumer flow.
@@ -54,6 +74,7 @@ module Karafka
         'error.occurred',
         error: e,
         caller: self,
+        seek_offset: coordinator.seek_offset,
         type: 'consumer.consume.error'
       )
     ensure
@@ -65,32 +86,21 @@ module Karafka
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api.
     def on_after_consume
-      return if revoked?
-
-      if coordinator.success?
-        coordinator.pause_tracker.reset
-
-        # Mark as consumed only if manual offset management is not on
-        return if topic.manual_offset_management?
-
-        # We use the non-blocking one here. If someone needs the blocking one, can implement it
-        # with manual offset management
-        mark_as_consumed(messages.last)
-      else
-        pause(coordinator.seek_offset)
-      end
+      handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.after_consume.error'
+      )
     end
 
     # Trigger method for running on partition revocation.
     #
     # @private
     def on_revoked
-      # We need to always un-pause the processing in case we have lost a given partition.
-      # Otherwise the underlying librdkafka would not know we may want to continue processing and
-      # the pause could in theory last forever
-      resume
-
-      coordinator.revoke
+      handle_revoked
 
       Karafka.monitor.instrument('consumer.revoked', caller: self) do
         revoked

data/lib/karafka/connection/listener.rb

@@ -46,8 +46,8 @@ module Karafka
         Karafka.monitor.instrument(
           'connection.listener.before_fetch_loop',
           caller: self,
-          subscription_group: @subscription_group,
-          client: @client
+          client: @client,
+          subscription_group: @subscription_group
         )
 
         fetch_loop
@@ -67,7 +67,6 @@ module Karafka
           @stopped = true
           @executors.clear
           @coordinators.reset
-          @client.commit_offsets!
           @client.stop
         end
       end
@@ -87,7 +86,8 @@ module Karafka
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,
-            client: @client
+            client: @client,
+            subscription_group: @subscription_group
           )
 
           resume_paused_partitions
@@ -95,6 +95,8 @@ module Karafka
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop.received',
             caller: self,
+            client: @client,
+            subscription_group: @subscription_group,
             messages_buffer: @messages_buffer
           ) do
             # We need to fetch data before we revoke lost partitions details as during the polling

data/lib/karafka/contracts/consumer_group.rb

@@ -24,20 +24,6 @@ module Karafka
 
         [[%i[topics], :names_not_unique]]
       end
-
-      virtual do |data, errors|
-        next unless errors.empty?
-
-        fetched_errors = []
-
-        data.fetch(:topics).each do |topic|
-          ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
-            fetched_errors << [[topic, key].flatten, value]
-          end
-        end
-
-        fetched_errors
-      end
     end
   end
 end

data/lib/karafka/contracts/{consumer_group_topic.rb → topic.rb}

@@ -3,13 +3,13 @@
 module Karafka
   module Contracts
     # Consumer group topic validation rules.
-    class ConsumerGroupTopic < Base
+    class Topic < Base
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
             File.join(Karafka.gem_root, 'config', 'errors.yml')
           )
-        ).fetch('en').fetch('validations').fetch('consumer_group_topic')
+        ).fetch('en').fetch('validations').fetch('topic')
       end
 
       required(:consumer) { |val| !val.nil? }
@@ -19,7 +19,6 @@ module Karafka
       required(:max_messages) { |val| val.is_a?(Integer) && val >= 1 }
       required(:initial_offset) { |val| %w[earliest latest].include?(val) }
       required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
-      required(:manual_offset_management) { |val| [true, false].include?(val) }
       required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:subscription_group) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
 

data/lib/karafka/errors.rb

@@ -41,11 +41,13 @@ module Karafka
     # Raised when the license token is not valid
     InvalidLicenseTokenError = Class.new(BaseError)
 
-    # Used to instrument this error into the error notifications
-    # We do not raise it so we won't crash deployed systems
-    ExpiredLicenseTokenError = Class.new(BaseError)
+    # This should never happen. Please open an issue if it does.
+    InvalidCoordinatorStateError = Class.new(BaseError)
+
+    # This should never happen. Please open an issue if it does.
+    StrategyNotFoundError = Class.new(BaseError)
 
     # This should never happen. Please open an issue if it does.
-    InvalidCoordinatorState = Class.new(BaseError)
+    SkipMessageNotFoundError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -14,6 +14,8 @@ module Karafka
         fatal
       ].freeze
 
+      private_constant :USED_LOG_LEVELS
+
       # Logs each messages fetching attempt
       #
       # @param event [Dry::Events::Event] event details including payload
@@ -85,13 +87,6 @@ module Karafka
         end
       end
 
-      # Logs info that we're initializing Karafka app.
-      #
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_initializing(_event)
-        info 'Initializing Karafka framework'
-      end
-
       # Logs info that we're running Karafka app.
       #
       # @param _event [Dry::Events::Event] event details including payload
@@ -118,6 +113,19 @@ module Karafka
         info 'Stopped Karafka server'
       end
 
+      # Logs info when we have dispatched a message the the DLQ
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_dead_letter_queue_dispatched(event)
+        message = event[:message]
+        offset = message.offset
+        topic = event[:caller].topic.name
+        dlq_topic = event[:caller].topic.dead_letter_queue.topic
+        partition = message.partition
+
+        info "Dispatched message #{offset} from #{topic}/#{partition} to DQL topic: #{dlq_topic}"
+      end
+
       # There are many types of errors that can occur in many places, but we provide a single
       # handler for all of them to simplify error instrumentation.
       # @param event [Dry::Events::Event] event details including payload
@@ -133,6 +141,15 @@ module Karafka
         when 'consumer.revoked.error'
           error "Consumer on revoked failed due to an error: #{error}"
           error details
+        when 'consumer.before_enqueue.error'
+          error "Consumer before enqueue failed due to an error: #{error}"
+          error details
+        when 'consumer.before_consume.error'
+          error "Consumer before consume failed due to an error: #{error}"
+          error details
+        when 'consumer.after_consume.error'
+          error "Consumer after consume failed due to an error: #{error}"
+          error details
         when 'consumer.shutdown.error'
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
@@ -142,9 +159,6 @@ module Karafka
         when 'connection.listener.fetch_loop.error'
           error "Listener fetch loop error: #{error}"
           error details
-        when 'licenser.expired'
-          error error
-          error details
         when 'runner.call.error'
           fatal "Runner crashed due to an error: #{error}"
           fatal details
@@ -154,7 +168,7 @@ module Karafka
           error "librdkafka internal error occurred: #{error}"
           error details
         # Those will only occur when retries in the client fail and when they did not stop after
-        # backoffs
+        # back-offs
         when 'connection.client.poll.error'
           error "Data polling error occurred: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -32,6 +32,8 @@ module Karafka
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
+        dead_letter_queue.dispatched
+
         worker.process
         worker.processed
         worker.completed