karafka 2.3.1 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d28565a68d29a1f8f14be1878bf3a1f41611028069586fb4ece1572b8eb72eb
-  data.tar.gz: 8c7f6ea1ec67e07ed107c7347d594a26ba923cecc7282c346eb24863e0324412
+  metadata.gz: 707ae730a3d16f7daa69b4d7fc91f27420e408f0c441aee1b3072a45a03ab18b
+  data.tar.gz: a06597a2361af03cbca21f98a4efc339c0cf582bf21a5794a8b9db1179c8b6c9
 SHA512:
-  metadata.gz: 73d43079b8e83babe2f722acdd11a67d6a5c361998ffada9b46b0f28b3ebc9c430d7b28cf56cb2aba808e25c234fffc7b2ee945e9e265160fb7cba6f821fcc70
-  data.tar.gz: 465b1088763ddc3ea63e53ef92765ae5719d6d1a83522288976321d1a64d24616b06d6571ce11425fd8a0dbf36d8c595871b92abfc8de8bcecc444181afe5123
+  metadata.gz: dbe0771e0764a74a7e86870c208b5c845fbfe9f022e9692db7dc1efe2e95ba65cb79f2a771c249e161cd2e540c2e18e4c4af72419849481a97e01ec70112a0bb
+  data.tar.gz: 9c6e32a5cc8fa7a82f8f0b9d50510863767eb11bfb57a4c356eb836d497e31981b0e0a19c814dfc438f3520320dd2eb973bf9c362cccec994bb4fa6dd0db1e27

data/.rspec

@@ -3,3 +3,5 @@
 --exclude-pattern "spec/integrations/**/*_spec.rb"
 # Do not run pro at the same time as default specs as it would cause state conflicts
 --tag ~type:pro
+# Do not run fork because they need to run in isolation
+--tag ~mode:fork

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Karafka framework changelog
 
+## 2.3.3 (2024-02-26)
+- [Enhancement] Routing based topics allocation for swarm (Pro)
+- [Enhancement] Publish the `-1` shutdown reason status for a non-responding node in swarm.
+- [Enhancement] Allow for using the `distribution` mode for DataDog listener histogram reporting (Aerdayne).
+- [Change] Change `internal.swarm.node_report_timeout` to 60 seconds from 30 seconds to compensate for long pollings.
+- [Fix] Static membership routing evaluation happens too early in swarm.
+- [Fix] Close producer in supervisor prior to forking and warmup to prevent invalid memory states.
+
+## 2.3.2 (2024-02-16)
+- **[Feature]** Provide swarm capabilities to OSS and Pro.
+- **[Feature]** Provide ability to use complex strategies in DLQ (Pro).
+- [Enhancement] Support using `:partition` as the partition key for ActiveJob assignments.
+- [Enhancement] Expand Logger listener with swarm notifications.
+- [Enhancement] Introduce K8s swarm liveness listener.
+- [Enhancement] Use `Process.warmup` in Ruby 3.3+ prior to forks (in swarm) and prior to app start.
+- [Enhancement] Provide `app.before_warmup` event to allow hooking code loading tools prior to final warmup.
+- [Enhancement] Provide `Consumer#errors_tracker` to be able to get errors that occurred while doing complex recovery.
+- [Fix] Infinite consecutive error flow with VPs and without DLQ can cause endless offsets accumulation.
+- [Fix] Quieting mode causes too early unsubscribe.
+
 ## 2.3.1 (2024-02-08)
 - [Refactor] Ensure that `Karafka::Helpers::Async#async_call` can run from multiple threads.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.1)
+    karafka (2.3.3)
       karafka-core (>= 2.3.0, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -41,11 +41,11 @@ GEM
       concurrent-ruby (~> 1.0)
     karafka-core (2.3.0)
       karafka-rdkafka (>= 0.14.8, < 0.15.0)
-    karafka-rdkafka (0.14.9)
+    karafka-rdkafka (0.14.10)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.8.1)
+    karafka-web (0.8.2)
       erubi (~> 1.4)
       karafka (>= 2.3.0, < 2.4.0)
       karafka-core (>= 2.3.0, < 2.4.0)
@@ -56,7 +56,7 @@ GEM
     mutex_m (0.2.0)
     rack (3.0.9)
     rake (13.1.0)
-    roda (3.76.0)
+    roda (3.77.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -81,10 +81,10 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.13)
+    waterdrop (2.6.14)
       karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.12)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby

data/README.md

@@ -9,13 +9,13 @@
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
 - Has a built-in [Web UI](https://karafka.io/docs/Web-UI-Features/) providing a convenient way to monitor and manage Karafka-based applications.
-- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
+- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work) and [processes](https://karafka.io/docs/Swarm-Multi-Process).
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
 - Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
 - Has a seamless [Dead Letter Queue](https://karafka.io/docs/Dead-Letter-Queue/) functionality built-in
 - Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
-- Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.
+- Has an out-of the box [AppSignal](https://karafka.io/docs/Monitoring-and-Logging/#appsignal-metrics-and-error-tracking) and [StatsD/DataDog](https://karafka.io/docs/Monitoring-and-Logging/#datadog-and-statsd-integration) monitoring with dashboard templates.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb

data/bin/integrations

@@ -43,7 +43,8 @@ class Scenario
     'consumption/worker_critical_error_behaviour_spec.rb' => [0, 2].freeze,
     'shutdown/on_hanging_jobs_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
-    'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze
+    'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
+    'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES

data/bin/rspecs

@@ -2,5 +2,9 @@
 
 set -e
 
-SPECS_TYPE=regular bundle exec rspec --tag ~type:pro
-SPECS_TYPE=pro bundle exec rspec --tag type:pro
+# Run only regular non-forking specs first
+SPECS_TYPE=regular bundle exec rspec --tag ~type:pro --tag ~mode:fork
+# Run forking specs, they need to run in isolation not to crash because of librdkafka
+SPECS_TYPE=regular bundle exec rspec --tag mode:fork
+# Run pro specs at the end
+SPECS_TYPE=pro bundle exec rspec --tag type:pro --tag ~mode:fork

data/config/locales/errors.yml

@@ -1,11 +1,12 @@
 en:
   validations:
     config:
-      missing: needs to be present
-      client_id_format: 'needs to be a string with a Kafka accepted format'
       license.entity_format: needs to be a string
       license.token_format: needs to be either false or a string
       license.expires_on_format: needs to be a valid date
+
+      missing: needs to be present
+      client_id_format: 'needs to be a string with a Kafka accepted format'
       concurrency_format: needs to be an integer bigger than 0
       consumer_mapper_format: needs to be present
       consumer_persistence_format: needs to be either true or false
@@ -14,7 +15,14 @@ en:
       pause_with_exponential_backoff_format: needs to be either true or false
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
+      max_wait_time_max_wait_time_vs_swarm_node_report_timeout: >
+        cannot be more than 80% of internal.swarm.node_report_timeout.
+        Decrease max_wait_time or increase node_report_timeout
       kafka_format: needs to be a filled hash
+      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+      max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
+      shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+
       internal.processing.jobs_builder_format: cannot be nil
       internal.processing.jobs_queue_class_format: cannot be nil
       internal.processing.scheduler_class_format: cannot be nil
@@ -22,14 +30,20 @@ en:
       internal.processing.partitioner_class_format: cannot be nil
       internal.processing.strategy_selector_format: cannot be nil
       internal.processing.expansions_selector_format: cannot be nil
-      internal.active_job.dispatcher: cannot be nil
-      internal.active_job.job_options_contract: cannot be nil
-      internal.active_job.consumer_class: cannot be nil
+
+      internal.active_job.dispatcher_format: cannot be nil
+      internal.active_job.job_options_contract_format: cannot be nil
+      internal.active_job.consumer_class_format: cannot be nil
+
       internal.status_format: needs to be present
       internal.process_format: needs to be present
       internal.tick_interval_format: needs to be an integer bigger or equal to 1000
+      internal.supervision_sleep_format: needs to be an integer bigger than 0
+      internal.forceful_exit_code_format: needs to be an integer bigger or equal to 0
+
       internal.routing.builder_format: needs to be present
       internal.routing.subscription_groups_builder_format: needs to be present
+
       internal.connection.manager_format: needs to be present
       internal.connection.conductor_format: needs to be present
       internal.connection.proxy.query_watermark_offsets.timeout_format: needs to be an integer bigger than 0
@@ -41,14 +55,25 @@ en:
       internal.connection.proxy.committed.timeout_format: needs to be an integer bigger than 0
       internal.connection.proxy.committed.max_attempts_format: needs to be an integer bigger than 0
       internal.connection.proxy.committed.wait_time_format: needs to be an integer bigger than 0
-      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
-      max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
-      shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+
+      internal.swarm.manager_format: cannot be nil
+      internal.swarm.orphaned_exit_code_format: needs to be an integer bigger or equal to 0
+      internal.swarm.pidfd_open_syscall_format: needs to be an integer bigger or equal to 0
+      internal.swarm.pidfd_signal_syscall_format: needs to be an integer bigger or equal to 0
+      internal.swarm.supervision_interval_format: needs to be an integer bigger or equal to 1000
+      internal.swarm.liveness_interval_format: needs to be an integer bigger or equal to 1000
+      internal.swarm.liveness_listener_format: cannot be nil
+      internal.swarm.node_report_timeout_format: needs to be an integer bigger or equal to 1000
+      internal.swarm.node_restart_timeout_format: needs to be an integer bigger or equal to 1000
+
       admin.kafka_format: needs to be a hash
       admin.group_id_format: 'needs to be a string with a Kafka accepted format'
       admin.max_wait_time_format: 'needs to be an integer bigger than 0'
       admin.max_attempts_format: 'needs to be an integer bigger than 0'
 
+      swarm.nodes_format: 'needs to be an integer bigger than 0'
+      swarm.node_format: needs to be false or node instance
+
     server_cli_options:
       missing: needs to be present
       consumer_groups_inclusion: Unknown consumer group name

data/config/locales/pro_errors.yml

@@ -8,6 +8,8 @@ en:
       long_running_job.active_format: needs to be either true or false
 
       dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
+      dead_letter_queue.strategy_format: 'needs to respond to #call'
+      dead_letter_queue.strategy_missing: needs to be present
 
       throttling.active_format: needs to be either true or false
       throttling.limit_format: needs to be equal or more than 1
@@ -56,6 +58,10 @@ en:
       subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
       subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
 
+      swarm.active_format: needs to be true
+      swarm.nodes_format: needs to be a range or an array of nodes ids
+      swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.3
+    image: confluentinc/cp-kafka:7.6.0
 
     ports:
       - 9092:9092

data/lib/karafka/app.rb

@@ -6,6 +6,20 @@ module Karafka
     extend Setup::Dsl
 
     class << self
+      # Notifies the Ruby virtual machine that the boot sequence is finished, and that now is a
+      # good time to optimize the application. In case of older Ruby versions, runs compacting,
+      # which is part of the full warmup introduced in Ruby 3.3.
+      def warmup
+        # Per recommendation, this should not run in children nodes
+        return if Karafka::App.config.swarm.node
+
+        monitor.instrument('app.before_warmup', caller: self)
+
+        return GC.compact unless ::Process.respond_to?(:warmup)
+
+        ::Process.warmup
+      end
+
       # @return [Karafka::Routing::Builder] consumers builder instance alias
       def consumer_groups
         config

data/lib/karafka/cli/base.rb

@@ -5,6 +5,8 @@ module Karafka
     # Base class for all the command that we want to define
     # This base class provides an interface to easier separate single independent commands
     class Base
+      include Helpers::Colorize
+
       # @return [Hash] given command cli options
       attr_reader :options
 
@@ -19,6 +21,23 @@ module Karafka
         raise NotImplementedError, 'Implement this in a subclass'
       end
 
+      private
+
+      # Prints marketing info
+      def print_marketing_info
+        Karafka.logger.info Info::BANNER
+
+        if Karafka.pro?
+          Karafka.logger.info(
+            green('Thank you for using Karafka Pro!')
+          )
+        else
+          Karafka.logger.info(
+            red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
+          )
+        end
+      end
+
       class << self
         # Loads proper environment with what is needed to run the CLI
         def load

data/lib/karafka/cli/server.rb

@@ -5,8 +5,6 @@ module Karafka
   class Cli
     # Server Karafka Cli action
     class Server < Base
-      include Helpers::Colorize
-
       # Types of things we can include / exclude from the routing via the CLI options
       SUPPORTED_TYPES = ::Karafka::Routing::ActivityManager::SUPPORTED_TYPES
 
@@ -16,63 +14,68 @@ module Karafka
 
       aliases :s, :consumer
 
-      option(
-        :consumer_groups,
-        'Runs server only with specified consumer groups',
-        Array,
-        %w[
-          -g
-          --consumer_groups
-          --include_consumer_groups
-        ]
-      )
-
-      option(
-        :subscription_groups,
-        'Runs server only with specified subscription groups',
-        Array,
-        %w[
-          --subscription_groups
-          --include_subscription_groups
-        ]
-      )
-
-      option(
-        :topics,
-        'Runs server only with specified topics',
-        Array,
-        %w[
-          --topics
-          --include_topics
-        ]
-      )
-
-      option(
-        :exclude_consumer_groups,
-        'Runs server without specified consumer groups',
-        Array,
-        %w[
-          --exclude_consumer_groups
-        ]
-      )
-
-      option(
-        :exclude_subscription_groups,
-        'Runs server without specified subscription groups',
-        Array,
-        %w[
-          --exclude_subscription_groups
-        ]
-      )
-
-      option(
-        :exclude_topics,
-        'Runs server without specified topics',
-        Array,
-        %w[
-          --exclude_topics
-        ]
-      )
+      # Those options can also be used when in swarm mode, hence we re-use
+      OPTIONS_BUILDER = lambda do
+        option(
+          :consumer_groups,
+          'Runs server only with specified consumer groups',
+          Array,
+          %w[
+            -g
+            --consumer_groups
+            --include_consumer_groups
+          ]
+        )
+
+        option(
+          :subscription_groups,
+          'Runs server only with specified subscription groups',
+          Array,
+          %w[
+            --subscription_groups
+            --include_subscription_groups
+          ]
+        )
+
+        option(
+          :topics,
+          'Runs server only with specified topics',
+          Array,
+          %w[
+            --topics
+            --include_topics
+          ]
+        )
+
+        option(
+          :exclude_consumer_groups,
+          'Runs server without specified consumer groups',
+          Array,
+          %w[
+            --exclude_consumer_groups
+          ]
+        )
+
+        option(
+          :exclude_subscription_groups,
+          'Runs server without specified subscription groups',
+          Array,
+          %w[
+            --exclude_subscription_groups
+          ]
+        )
+
+        option(
+          :exclude_topics,
+          'Runs server without specified topics',
+          Array,
+          %w[
+            --exclude_topics
+          ]
+        )
+      end
+
+      instance_exec(&OPTIONS_BUILDER)
 
       # Start the Karafka server
       def call
@@ -85,8 +88,6 @@ module Karafka
         Karafka::Server.run
       end
 
-      private
-
       # Registers things we want to include (if defined)
       def register_inclusions
         activities = ::Karafka::App.config.internal.routing.activity_manager
@@ -108,21 +109,6 @@ module Karafka
           names.each { |name| activities.exclude(type, name) }
         end
       end
-
-      # Prints marketing info
-      def print_marketing_info
-        Karafka.logger.info Info::BANNER
-
-        if Karafka.pro?
-          Karafka.logger.info(
-            green('Thank you for using Karafka Pro!')
-          )
-        else
-          Karafka.logger.info(
-            red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
-          )
-        end
-      end
     end
   end
 end

data/lib/karafka/cli/swarm.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka framework Cli
+  class Cli
+    # Starts swarm of consumers forked from the supervisor
+    class Swarm < Base
+      desc 'Starts swarm of Karafka consumers with a supervisor'
+
+      aliases :swarm
+
+      instance_exec(&Server::OPTIONS_BUILDER)
+
+      # Starts the swarm
+      def call
+        ::Karafka::Swarm.ensure_supported!
+
+        # Print our banner and info in the dev mode
+        print_marketing_info if Karafka::App.env.development?
+
+        # This will register inclusions and exclusions in the routing, so all forks will use it
+        server = Server.new
+        server.register_inclusions
+        server.register_exclusions
+
+        Karafka::Swarm::Supervisor.new.run
+      end
+    end
+  end
+end

data/lib/karafka/connection/client.rb

@@ -600,6 +600,13 @@ module Karafka
       # @return [Rdkafka::Consumer]
       def build_consumer
         ::Rdkafka::Config.logger = ::Karafka::App.config.logger
+
+        # We need to refresh the setup of this subscription group in case we started running in a
+        # swarm. The initial configuration for validation comes from the parent node, but it needs
+        # to be altered in case of a static group membership usage for correct mapping of the
+        # group instance id.
+        @subscription_group.refresh
+
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
         config.consumer_rebalance_listener = @rebalance_callback
         # We want to manage the events queue independently from the messages queue. Thanks to that

data/lib/karafka/constraints.rb

@@ -15,13 +15,13 @@ module Karafka
         # Skip verification if web is not used at all
         return unless require_version('karafka/web')
 
-        # All good if version higher than 0.7.x because we expect 0.8.0 or higher
-        return if version(Karafka::Web::VERSION) >= version('0.7.100')
+        # All good if version higher than 0.8.1 because we expect 0.8.2 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.8.2')
 
         # If older web-ui used, we cannot allow it
         raise(
           Errors::DependencyConstraintsError,
-          'karafka-web < 0.8.0 is not compatible with this karafka version'
+          'karafka-web < 0.8.2 is not compatible with this karafka version'
         )
       end
 

data/lib/karafka/contracts/config.rb

@@ -34,6 +34,11 @@ module Karafka
       required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
 
+      nested(:swarm) do
+        required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
+        required(:node) { |val| val == false || val.is_a?(Karafka::Swarm::Node) }
+      end
+
       nested(:admin) do
         # Can be empty because inherits values from the root kafka
         required(:kafka) { |val| val.is_a?(Hash) }
@@ -49,6 +54,20 @@ module Karafka
         # In theory this could be less than a second, however this would impact the maximum time
         # of a single consumer queue poll, hence we prevent it
         required(:tick_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+        required(:supervision_sleep) { |val| val.is_a?(Numeric) && val.positive? }
+        required(:forceful_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
+
+        nested(:swarm) do
+          required(:manager) { |val| !val.nil? }
+          required(:orphaned_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
+          required(:pidfd_open_syscall) { |val| val.is_a?(Integer) && val >= 0 }
+          required(:pidfd_signal_syscall) { |val| val.is_a?(Integer) && val >= 0 }
+          required(:supervision_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:liveness_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:liveness_listener) { |val| !val.nil? }
+          required(:node_report_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:node_restart_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
+        end
 
         nested(:connection) do
           required(:manager) { |val| !val.nil? }
@@ -143,6 +162,28 @@ module Karafka
 
         [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
       end
+
+      # `internal.swarm.node_report_timeout` should not be close to `max_wait_time` otherwise
+      # there may be a case where node cannot report often enough because it is clogged by waiting
+      # on more data.
+      #
+      # We handle that at a config level to make sure that this is correctly configured.
+      #
+      # We do not validate this in the context of swarm usage (validate only if...) because it is
+      # often that swarm only runs on prod and we do not want to crash it surprisingly.
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        max_wait_time = data.fetch(:max_wait_time)
+        node_report_timeout = data.fetch(:internal)[:swarm][:node_report_timeout] || false
+
+        next unless node_report_timeout
+        # max wait time should be at least 20% smaller than the reporting time to have enough
+        # time for reporting
+        next if max_wait_time < node_report_timeout * 0.8
+
+        [[%i[max_wait_time], :max_wait_time_vs_swarm_node_report_timeout]]
+      end
     end
   end
 end

data/lib/karafka/errors.rb

@@ -87,5 +87,17 @@ module Karafka
     # Because we do not want to require web out of the box and we do not want to lock web with
     # karafka 1:1, we do such a sanity check
     DependencyConstraintsError = Class.new(BaseError)
+
+    # Raised when we were not able to open pidfd for given pid
+    # This should not happen. If you see it, please report.
+    PidfdOpenFailedError = Class.new(BaseError)
+
+    # Failed to send signal to a process via pidfd
+    # This should not happen. If you see it, please report.
+    PidfdSignalFailedError = Class.new(BaseError)
+
+    # Raised when given option/feature is not supported on a given platform or when given option
+    # is not supported in a given configuration
+    UnsupportedOptionError = Class.new(BaseError)
   end
 end

data/lib/karafka/helpers/config_importer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Module allowing for configuration injections. By default injects whole app config
+    # Allows for granular config injection
+    class ConfigImporter < Module
+      # @param attributes [Hash<Symbol, Array<Symbol>>] map defining what we want to inject.
+      #   The key is the name under which attribute will be visible and the value is the full
+      #   path to the attribute
+      def initialize(attributes = { config: %i[itself] })
+        super()
+        @attributes = attributes
+      end
+
+      # @param model [Object] object to which we want to add the config fetcher
+      def included(model)
+        super
+
+        @attributes.each do |name, path|
+          model.class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            def #{name}
+              @#{name} ||= ::Karafka::App.config.#{path.join('.')}
+            end
+          RUBY
+        end
+      end
+    end
+  end
+end