karafka 2.0.0.rc3 → 2.0.0.rc6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2a60c5c750ef681042f6e0a0d61032c2b9d9e8d48fa73ec480157c7bffb4cc7
-  data.tar.gz: c04074859408e77d236e86dcfdbe8502ba5322f6e7d8d1e236c9aa838b3e1a29
+  metadata.gz: 13fbbd2c70987d84f768acb1a01f02f18ca59f49d8f02a2c1103b6870dbcfb15
+  data.tar.gz: 223187a692ae8e3d5da3c028c135c20a3de0c05b26b7f67cfbae11b928b02773
 SHA512:
-  metadata.gz: 487dc8f5fb131dc8f25087ae026b40710c9859e7849b878ac42053737dd665a0147462fd6348e3e03eb87e0d6657c8f0038b02506f525ee221ee1f6d806bcf84
-  data.tar.gz: 23a68b2d97cd5441088521c949c2a0ec23befde2fa81c9b9178ea6786a582aea5e342697d08f9a3dd1dfe1401792bcbc3e936b91c87eb740ef66beccd69b39bf
+  metadata.gz: 381ddd6a4d3f9695ca5d657d4960213afb3c5b92b8b4e0f51b45f75d1de92f65a570ec903abb43d5cb2cf371a557dbbb01cb78ce33eb118902e6d5e77fdb6095
+  data.tar.gz: e4675ba160b9443240497f874e29bb7614b77bf06c4fa8e768aff42060de182059ac4a26b07841d251f5bb4d257ad8f3920d77ec551c8f1487a4eef3e6a6db32

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Karafka framework changelog
 
+## 2.0.0.rc6 (2022-08-05)
+- Update licenser to use a gem based approach based on `karafka/pro/license`.
+- Do not mark intermediate jobs as consumed when Karafka runs Enhanced Active Job with Virtual Partitions.
+- Improve development experience by adding fast cluster state changes refresh (#944)
+- Improve the license loading.
+
+## 2.0.0.rc5 (2022-08-01)
+- Improve specs stability
+- Improve forceful shutdown
+- Add support for debug `TTIN` backtrace printing
+- Fix a case where logger listener would not intercept `warn` level
+- Require `rdkafka` >= `0.12`
+- Replace statistics decorator with the one from `karafka-core`
+
+## 2.0.0.rc4 (2022-07-28)
+- Remove `dry-monitor`
+- Use `karafka-core`
+- Improve forceful shutdown resources finalization
+- Cache consumer client name
+
 ## 2.0.0.rc3 (2022-07-26)
 - Fix Pro partitioner hash function may not utilize all the threads (#907).
 - Improve virtual partitions messages distribution.
@@ -65,8 +85,8 @@
 - Fix a case where consecutive CTRL+C (non-stop) would case an exception during forced shutdown
 - Add missing `consumer.prepared.error` into `LoggerListener`
 - Delegate partition resuming from the consumers to listeners threads.
-- Add support for Long Running Jobs (LRJ) for ActiveJob [PRO]
-- Add support for Long Running Jobs for consumers [PRO]
+- Add support for Long-Running Jobs (LRJ) for ActiveJob [PRO]
+- Add support for Long-Running Jobs for consumers [PRO]
 - Allow `active_job_topic` to accept a block for extra topic related settings
 - Remove no longer needed logger threads
 - Auto-adapt number of processes for integration specs based on the number of CPUs

data/CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contribute
+# Contributing
 
 ## Introduction
 
@@ -8,11 +8,7 @@ We welcome any type of contribution, not only code. You can help with:
 - **QA**: file bug reports, the more details you can give the better (e.g. screenshots with the console open)
 - **Marketing**: writing blog posts, howto's, printing stickers, ...
 - **Community**: presenting the project at meetups, organizing a dedicated meetup for the local community, ...
-- **Code**: take a look at the [open issues](issues). Even if you can't write code, commenting on them, showing that you care about a given issue matters. It helps us triage them.
-
-## Your First Contribution
-
-Working on your first Pull Request? You can learn how from this *free* series, [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github).
+- **Code**: take a look at the [open issues](https://github.com/karafka/karafka/issues). Even if you can't write code, commenting on them, showing that you care about a given issue matters. It helps us triage them.
 
 ## Submitting code
 
@@ -32,5 +28,5 @@ By sending a pull request to the pro components, you are agreeing to transfer th
 
 ## Questions
 
-If you have any questions, create an [issue](issue) (protip: do a quick search first to see if someone else didn't ask the same question before!).
-You can also reach us at hello@karafka.opencollective.com.
+If you have any questions, create an [issue](https://github.com/karafka/karafka/issues) (protip: do a quick search first to see if someone else didn't ask the same question before!).
+You can also reach us at contact@karafka.io.

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.rc3)
-      dry-monitor (~> 0.5)
-      rdkafka (>= 0.10)
+    karafka (2.0.0.rc6)
+      karafka-core (>= 2.0.2, < 3.0.0)
+      rdkafka (>= 0.12)
       thor (>= 0.20)
-      waterdrop (>= 2.3.3, < 3.0.0)
+      waterdrop (>= 2.4.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -23,19 +23,6 @@ GEM
     concurrent-ruby (1.1.10)
     diff-lcs (1.5.0)
     docile (1.4.0)
-    dry-configurable (0.15.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.6)
-    dry-core (0.8.0)
-      concurrent-ruby (~> 1.0)
-    dry-events (0.3.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5)
-    dry-monitor (0.6.1)
-      dry-configurable (~> 0.13, >= 0.13.0)
-      dry-core (~> 0.5, >= 0.5)
-      dry-events (~> 0.2)
-      zeitwerk (~> 2.5)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
@@ -43,6 +30,8 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
+    karafka-core (2.0.2)
+      concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.2)
     rake (13.0.6)
@@ -70,11 +59,10 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
-    tzinfo (2.0.4)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.3.3)
-      concurrent-ruby (>= 1.1)
-      dry-monitor (~> 0.5)
+    waterdrop (2.4.1)
+      karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.0)

data/LICENSE-COMM

@@ -6,7 +6,7 @@ IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMEN
 
 ------------------------------------------------------------------------------
 
-In order to use the Software under this Agreement, you must receive an application token at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement.
+In order to use the Software under this Agreement, you must receive a "Source URL" to a license package at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement.
 
 1. License Grant
 
@@ -22,7 +22,7 @@ In order to use the Software under this Agreement, you must receive an applicati
 
 3. Restricted Uses.
 
-3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Maciej Mensfeld; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of token verification or copy protection used by Maciej Mensfeld in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Maciej Mensfeld; (j) use the Software to develop a product which is competitive with any Maciej Mensfeld product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Maciej Mensfeld in writing. If your unique application token is ever published, Maciej Mensfeld reserves the right to terminate your access without notice.
+3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Maciej Mensfeld; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Maciej Mensfeld in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Maciej Mensfeld; (j) use the Software to develop a product which is competitive with any Maciej Mensfeld product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Maciej Mensfeld in writing. If your unique Source URL is ever published, Maciej Mensfeld reserves the right to terminate your access without notice.
 
 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
 

data/README.md

@@ -4,18 +4,18 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0.0.rc2` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
+**Note**: All of the documentation here refers to Karafka `2.0.0.rc5` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
 
 ## About Karafka
 
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
-- Supports parallel processing in [multiple threads](Concurrency-and-multithreading) (also for a single topic partition work)
-- Has [ActiveJob backend](Active-Job) support (including ordered jobs)
-- [Automatically integrates](Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails=) with Ruby on Rails
-- Supports in-development [code reloading](Auto-reload-of-code-changes-in-development)
+- Supports parallel processing in [multiple threads](https://github.com/karafka/karafka/wiki/Concurrency-and-multithreading) (also for a [single topic partition](https://github.com/karafka/karafka/wiki/Pro-Virtual-Partitions) work)
+- Has [ActiveJob backend](https://github.com/karafka/karafka/wiki/Active-Job) support (including [ordered jobs](https://github.com/karafka/karafka/wiki/Pro-Enhanced-Active-Job#ordered-jobs))
+- [Automatically integrates](https://github.com/karafka/karafka/wiki/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails=) with Ruby on Rails
+- Supports in-development [code reloading](https://github.com/karafka/karafka/wiki/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
-- Has an out-of the box [StatsD/DataDog monitoring](Monitoring-and-logging) with a dashboard template.
+- Has an out-of the box [StatsD/DataDog monitoring](https://github.com/karafka/karafka/wiki/Monitoring-and-logging) with a dashboard template.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb
@@ -42,7 +42,7 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
 If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
 
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
-- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Ruby and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
+- [Kafka on Rails: Using Kafka with Ruby on Rails – Part 2 – Getting started with Rails and Kafka](https://mensfeld.pl/2018/01/kafka-on-rails-using-kafka-with-ruby-on-rails-part-2-getting-started-with-ruby-and-kafka/)
 
 If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to visit our [Getting started](https://github.com/karafka/karafka/wiki/Getting-started) guides and the [example apps repository](https://github.com/karafka/example-apps).
 
@@ -55,7 +55,7 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-bundle add karafka -v 2.0.0.rc2
+bundle add karafka -v 2.0.0.rc5
 
 bundle exec karafka install
 ```

data/bin/create_token

@@ -9,16 +9,10 @@ PRIVATE_KEY_LOCATION = File.join(Dir.home, '.ssh', 'karafka-pro', 'id_rsa')
 
 # Name of the entity that acquires the license
 ENTITY = ARGV[0]
-# Date till which license is valid
-EXPIRES_ON = Date.parse(ARGV[1])
 
 raise ArgumentError, 'Entity missing' if ENTITY.nil? || ENTITY.empty?
-raise ArgumentError, 'Expires on needs to be in the future' if EXPIRES_ON <= Date.today
 
-pro_token_data = {
-  entity: ENTITY,
-  expires_on: EXPIRES_ON
-}
+pro_token_data = { entity: ENTITY }
 
 # This code uses my private key to generate a new token for Karafka Pro capabilities
 private_key = OpenSSL::PKey::RSA.new(File.read(PRIVATE_KEY_LOCATION))

data/bin/integrations

@@ -22,7 +22,7 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
-MAX_BUFFER_OUTPUT = 10_240
+MAX_BUFFER_OUTPUT = 51_200
 
 # Abstraction around a single test scenario execution process
 class Scenario
@@ -37,7 +37,7 @@ class Scenario
     'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
     'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
-    'shutdown/on_hanging_poll_and_shutdown.rb' => [2].freeze
+    'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES

data/karafka.gemspec

@@ -12,17 +12,17 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[maciej@mensfeld.pl]
   spec.homepage    = 'https://karafka.io'
-  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails '
+  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
 
-  spec.add_dependency 'dry-monitor', '~> 0.5'
-  spec.add_dependency 'rdkafka', '>= 0.10'
+  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
+  spec.add_dependency 'rdkafka', '>= 0.12'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.3.3', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.6.0'
+  spec.required_ruby_version = '>= 2.7.0'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')

data/lib/karafka/cli/info.rb

@@ -52,8 +52,7 @@ module Karafka
         if Karafka.pro?
           [
             'License: Commercial',
-            "License entity: #{config.license.entity}",
-            "License expires on: #{config.license.expires_on}"
+            "License entity: #{config.license.entity}"
           ]
         else
           [

data/lib/karafka/connection/client.rb

@@ -275,6 +275,11 @@ module Karafka
 
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
+        # Once client is closed, we should not close it again
+        # This could only happen in case of a race-condition when forceful shutdown happens
+        # and triggers this from a different thread
+        return if @closed
+
         @mutex.synchronize do
           internal_commit_offsets(async: false)
 
@@ -340,6 +345,9 @@ module Karafka
         when :network_exception # 13
           reset
           return nil
+        when :unknown_topic_or_part
+          # This is expected and temporary until rdkafka catches up with metadata
+          return nil
         end
 
         raise if time_poll.attempts > MAX_POLL_RETRIES
@@ -359,7 +367,6 @@ module Karafka
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
         config.consumer_rebalance_listener = @rebalance_manager
         consumer = config.consumer
-        consumer.subscribe(*@subscription_group.topics.map(&:name))
         @name = consumer.name
 
         # Register statistics runner for this particular type of callbacks
@@ -384,6 +391,9 @@ module Karafka
           )
         )
 
+        # Subscription needs to happen after we assigned the rebalance callbacks just in case of
+        # a race condition
+        consumer.subscribe(*@subscription_group.topics.map(&:name))
         consumer
       end
 

data/lib/karafka/connection/listener.rb

@@ -34,6 +34,8 @@ module Karafka
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
         @messages_buffer = MessagesBuffer.new(subscription_group)
+        @mutex = Mutex.new
+        @stopped = false
       end
 
       # Runs the main listener fetch loop.
@@ -51,6 +53,25 @@ module Karafka
         fetch_loop
       end
 
+      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
+      # stops kafka client.
+      #
+      # @note This method is not private despite being part of the fetch loop because in case of
+      #   a forceful shutdown, it may be invoked from a separate thread
+      #
+      # @note We wrap it with a mutex exactly because of the above case of forceful shutdown
+      def shutdown
+        return if @stopped
+
+        @mutex.synchronize do
+          @stopped = true
+          @executors.clear
+          @coordinators.reset
+          @client.commit_offsets!
+          @client.stop
+        end
+      end
+
       private
 
       # Fetches the data and adds it to the jobs queue.
@@ -239,13 +260,6 @@ module Karafka
         @client.batch_poll until @jobs_queue.empty?(@subscription_group.id)
       end
 
-      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
-      # stops kafka client.
-      def shutdown
-        @client.commit_offsets!
-        @client.stop
-      end
-
       # We can stop client without a problem, as it will reinitialize itself when running the
       # `#fetch_loop` again. We just need to remember to also reset the runner as it is a long
       # running one, so with a new connection to Kafka, we need to initialize the state of the

data/lib/karafka/contracts/base.rb

@@ -3,7 +3,7 @@
 module Karafka
   module Contracts
     # Base contract for all Karafka contracts
-    class Base < ::WaterDrop::Contractable::Contract
+    class Base < ::Karafka::Core::Contractable::Contract
       # @param data [Hash] data for validation
       # @return [Boolean] true if all good
       # @raise [Errors::InvalidConfigurationError] invalid configuration error

data/lib/karafka/contracts/config.rb

@@ -21,7 +21,6 @@ module Karafka
       nested(:license) do
         required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
         required(:entity) { |val| val.is_a?(String) }
-        required(:expires_on) { |val| val.is_a?(Date) }
       end
 
       required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }

data/lib/karafka/errors.rb

@@ -24,9 +24,6 @@ module Karafka
     # Raised when we try to use Karafka CLI commands (except install) without a boot file
     MissingBootFileError = Class.new(BaseError)
 
-    # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEventError = Class.new(BaseError)
-
     # Raised when we've waited enough for shutting down a non-responsive process
     ForcefulShutdownError = Class.new(BaseError)
 

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -16,8 +16,7 @@ module Karafka
           @consumer_group_id = consumer_group_id
           @client_name = client_name
           @monitor = monitor
-          # We decorate both Karafka and WaterDrop statistics the same way
-          @statistics_decorator = ::WaterDrop::Instrumentation::Callbacks::StatisticsDecorator.new
+          @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
         end
 
         # Emits decorated statistics to the monitor

data/lib/karafka/instrumentation/logger_listener.rb

@@ -9,6 +9,7 @@ module Karafka
       USED_LOG_LEVELS = %i[
         debug
         info
+        warn
         error
         fatal
       ].freeze
@@ -60,11 +61,28 @@ module Karafka
         info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
-      # Logs info about system signals that Karafka received.
+      # Logs info about system signals that Karafka received and prints backtrace for threads in
+      # case of ttin
       #
       # @param event [Dry::Events::Event] event details including payload
       def on_process_notice_signal(event)
         info "Received #{event[:signal]} system signal"
+
+        # We print backtrace only for ttin
+        return unless event[:signal] == :SIGTTIN
+
+        # Inspired by Sidekiq
+        Thread.list.each do |thread|
+          tid = (thread.object_id ^ ::Process.pid).to_s(36)
+
+          warn "Thread TID-#{tid} #{thread['label']}"
+
+          if thread.backtrace
+            warn thread.backtrace.join("\n")
+          else
+            warn '<no backtrace available>'
+          end
+        end
       end
 
       # Logs info that we're initializing Karafka app.

data/lib/karafka/instrumentation/monitor.rb

@@ -1,66 +1,21 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Namespace for all the things related with Karafka instrumentation process
   module Instrumentation
-    # Monitor is used to hookup external monitoring services to monitor how Karafka works
-    # It provides a standardized API for checking incoming messages/enqueueing etc
-    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and NewRelic at the same
-    # time
-    # @note This class acts as a singleton because we are only permitted to have single monitor
-    #   per running process (just as logger)
-    class Monitor < Dry::Monitor::Notifications
-      # List of events that we support in the system and to which a monitor client can hook up
-      # @note The non-error once support timestamp benchmarking
-      # @note Depending on Karafka extensions and additional engines, this might not be the
-      #   complete list of all the events. Please use the #available_events on fully loaded
-      #   Karafka system to determine all of the events you can use.
-      BASE_EVENTS = %w[
-        app.initialized
-        app.running
-        app.stopping
-        app.stopped
-
-        consumer.consumed
-        consumer.revoked
-        consumer.shutdown
-
-        process.notice_signal
-
-        connection.listener.before_fetch_loop
-        connection.listener.fetch_loop
-        connection.listener.fetch_loop.received
-
-        worker.process
-        worker.processed
-
-        statistics.emitted
-
-        error.occurred
-      ].freeze
-
-      private_constant :BASE_EVENTS
-
-      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
-      def initialize
-        super(:karafka)
-        BASE_EVENTS.each(&method(:register_event))
-      end
-
-      # Allows us to subscribe to events with a code that will be yielded upon events
-      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
-      #   or a listener if we decide to go with object listener
-      def subscribe(event_name_or_listener)
-        return super unless event_name_or_listener.is_a?(String)
-        return super if available_events.include?(event_name_or_listener)
-
-        raise Errors::UnregisteredMonitorEventError, event_name_or_listener
-      end
-
-      # @return [Array<String>] names of available events to which we can subscribe
-      def available_events
-        __bus__.events.keys
+    # Karafka instrumentation monitor that we use to publish events
+    # By default uses our internal notifications bus but can be used with
+    # `ActiveSupport::Notifications` as well
+    class Monitor < ::Karafka::Core::Monitoring::Monitor
+      attr_reader :notifications_bus
+
+      # @param notifications_bus [Object] either our internal notifications bus or
+      #   `ActiveSupport::Notifications`
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = ::Karafka::Instrumentation::Notifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
       end
     end
   end

data/lib/karafka/instrumentation/notifications.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related with Karafka instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how Karafka works
+    # It provides a standardized API for checking incoming messages/enqueueing etc
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and NewRelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Notifications < Karafka::Core::Monitoring::Notifications
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      # @note Depending on Karafka extensions and additional engines, this might not be the
+      #   complete list of all the events. Please use the #available_events on fully loaded
+      #   Karafka system to determine all of the events you can use.
+      EVENTS = %w[
+        app.initialized
+        app.running
+        app.stopping
+        app.stopped
+
+        consumer.consumed
+        consumer.revoked
+        consumer.shutdown
+
+        process.notice_signal
+
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
+        connection.listener.fetch_loop.received
+
+        worker.process
+        worker.processed
+
+        statistics.emitted
+
+        error.occurred
+      ].freeze
+
+      private_constant :EVENTS
+
+      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super
+        EVENTS.each { |event| register_event(event) }
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -5,19 +5,19 @@ module Karafka
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
       # Updates proc title to an initializing one
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_initializing(_event)
         setproctitle('initializing')
       end
 
       # Updates proc title to a running one
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_running(_event)
         setproctitle('running')
       end
 
       # Updates proc title to a stopping one
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_stopping(_event)
         setproctitle('stopping')
       end

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -11,7 +11,7 @@ module Karafka
         #
         # @note You need to setup the `dogstatsd-ruby` client and assign it
         class Listener
-          include WaterDrop::Configurable
+          include ::Karafka::Core::Configurable
           extend Forwardable
 
           def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
@@ -65,7 +65,7 @@ module Karafka
 
           # Hooks up to WaterDrop instrumentation for emitted statistics
           #
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_statistics_emitted(event)
             statistics = event[:statistics]
 
@@ -76,7 +76,7 @@ module Karafka
 
           # Increases the errors count by 1
           #
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_error_occurred(event)
             extra_tags = ["type:#{event[:type]}"]
 
@@ -94,7 +94,7 @@ module Karafka
 
           # Reports how many messages we've polled and how much time did we spend on it
           #
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_connection_listener_fetch_loop_received(event)
             time_taken = event[:time]
             messages_count = event[:messages_buffer].size
@@ -105,7 +105,7 @@ module Karafka
 
           # Here we report majority of things related to processing as we have access to the
           # consumer
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_consumer_consumed(event)
             messages = event.payload[:caller].messages
             metadata = messages.metadata
@@ -124,7 +124,7 @@ module Karafka
             histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
           end
 
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_consumer_revoked(event)
             messages = event.payload[:caller].messages
             metadata = messages.metadata
@@ -137,7 +137,7 @@ module Karafka
             count('consumer.revoked', 1, tags: tags)
           end
 
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_consumer_shutdown(event)
             messages = event.payload[:caller].messages
             metadata = messages.metadata
@@ -151,7 +151,7 @@ module Karafka
           end
 
           # Worker related metrics
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_worker_process(event)
             jq_stats = event[:jobs_queue].statistics
 
@@ -162,7 +162,7 @@ module Karafka
 
           # We report this metric before and after processing for higher accuracy
           # Without this, the utilization would not be fully reflected
-          # @param event [Dry::Events::Event]
+          # @param event [Karafka::Core::Monitoring::Event]
           def on_worker_processed(event)
             jq_stats = event[:jobs_queue].statistics
 

data/lib/karafka/licenser.rb

@@ -8,8 +8,34 @@ module Karafka
 
     private_constant :PUBLIC_KEY_LOCATION
 
+    # Tries to prepare license and verifies it
+    #
+    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+    def prepare_and_verify(license_config)
+      prepare(license_config)
+      verify(license_config)
+    end
+
+    private
+
+    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+    def prepare(license_config)
+      # If there is token, no action needed
+      # We support a case where someone would put the token in instead of using one from the
+      # license. That's in case there are limitations to using external package sources, etc
+      return if license_config.token
+
+      begin
+        license_config.token || require('karafka-license')
+      rescue LoadError
+        return
+      end
+
+      license_config.token = Karafka::License.token
+    end
+
     # Check license and setup license details (if needed)
-    # @param license_config [Dry::Configurable::Config] config part related to the licensing
+    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
     def verify(license_config)
       # If no license, it will just run LGPL components without anything extra
       return unless license_config.token
@@ -29,19 +55,10 @@ module Karafka
       details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
 
       license_config.entity = details.fetch('entity')
-      license_config.expires_on = Date.parse(details.fetch('expires_on'))
-
-      return if license_config.expires_on > Date.today
-
-      raise_expired_license_token_in_dev(license_config.expires_on)
-
-      notify_if_license_expired(license_config.expires_on)
     end
 
-    private
-
     # Raises an error with info, that used token is invalid
-    # @param license_config [Dry::Configurable::Config]
+    # @param license_config [Karafka::Core::Configurable::Node]
     def raise_invalid_license_token(license_config)
       # We set it to false so `Karafka.pro?` method behaves as expected
       license_config.token = false
@@ -54,42 +71,5 @@ module Karafka
         MSG
       )
     end
-
-    # Raises an error for test and dev environments if running pro with expired license
-    # We never want to cause any non-dev problems and we should never crash anything else than
-    # tests and development envs.
-    #
-    # @param expires_on [Date] when the license expires
-    def raise_expired_license_token_in_dev(expires_on)
-      env = Karafka::App.env
-
-      return unless env.development? || env.test?
-
-      raise Errors::ExpiredLicenseTokenError.new, expired_message(expires_on)
-    end
-
-    # We do not raise an error here as we don't want to cause any problems to someone that runs
-    # Karafka on production. Error message is enough.
-    #
-    # @param expires_on [Date] when the license expires
-    def notify_if_license_expired(expires_on)
-      Karafka.logger.error(expired_message(expires_on))
-
-      Karafka.monitor.instrument(
-        'error.occurred',
-        caller: self,
-        error: Errors::ExpiredLicenseTokenError.new(expired_message(expires_on)),
-        type: 'licenser.expired'
-      )
-    end
-
-    # @param expires_on [Date] when the license expires
-    # @return [String] expired message
-    def expired_message(expires_on)
-      <<~MSG.tr("\n", ' ')
-        Your license expired on #{expires_on}.
-        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-      MSG
-    end
   end
 end

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -12,7 +12,7 @@ module Karafka
         # @note We need this to make sure that we allocate proper dispatched events only to
         #   callback listeners that should publish them
         def name
-          ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
+          @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
         end
       end
     end

data/lib/karafka/pro/active_job/consumer.rb

@@ -33,6 +33,10 @@ module Karafka
               ::ActiveSupport::JSON.decode(message.raw_payload)
             )
 
+            # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
+            # this could create random markings
+            next if topic.virtual_partitioner?
+
             mark_as_consumed(message)
           end
         end

data/lib/karafka/pro/base_consumer.rb

@@ -36,7 +36,7 @@ module Karafka
         end
       end
 
-      # Runs extra logic after consumption that is related to handling long running jobs
+      # Runs extra logic after consumption that is related to handling long-running jobs
       # @note This overwrites the '#on_after_consume' from the base consumer
       def on_after_consume
         coordinator.on_finished do |first_group_message, last_group_message|
@@ -59,7 +59,7 @@ module Karafka
           # Mark as consumed only if manual offset management is not on
           mark_as_consumed(last_message) unless topic.manual_offset_management? || revoked?
 
-          # If this is not a long running job there is nothing for us to do here
+          # If this is not a long-running job there is nothing for us to do here
           return unless topic.long_running_job?
 
           # Once processing is done, we move to the new offset based on commits

data/lib/karafka/pro/loader.rb

@@ -36,7 +36,7 @@ module Karafka
 
       class << self
         # Loads all the pro components and configures them wherever it is expected
-        # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
+        # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
         def setup(config)
           COMPONENTS.each { |component| require_relative(component) }

data/lib/karafka/pro/performance_tracker.rb

@@ -45,7 +45,7 @@ module Karafka
       end
 
       # @private
-      # @param event [Dry::Events::Event] event details
+      # @param event [Karafka::Core::Monitoring::Event] event details
       # Tracks time taken to process a single message of a given topic partition
       def on_consumer_consumed(event)
         consumer = event[:caller]

data/lib/karafka/pro/processing/coordinator.rb

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     module Processing

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -28,7 +28,7 @@ module Karafka
           virtual_partitioner != nil
         end
 
-        # @return [Boolean] is a given job on a topic a long running one
+        # @return [Boolean] is a given job on a topic a long-running one
         def long_running_job?
           @long_running_job || false
         end

data/lib/karafka/process.rb

@@ -9,6 +9,7 @@ module Karafka
       SIGINT
       SIGQUIT
       SIGTERM
+      SIGTTIN
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|

data/lib/karafka/server.rb

@@ -104,6 +104,9 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.each(&:terminate)
+        # We always need to shutdown clients to make sure we do not force the GC to close consumer.
+        # This can cause memory leaks and crashes.
+        listeners.each(&:shutdown)
 
         Karafka::App.producer.close
 

data/lib/karafka/setup/config.rb

@@ -12,14 +12,27 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend ::WaterDrop::Configurable
+      extend ::Karafka::Core::Configurable
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
         'client.id': 'karafka'
       }.freeze
 
-      private_constant :KAFKA_DEFAULTS
+      # Contains settings that should not be used in production but make life easier in dev
+      DEV_DEFAULTS = {
+        # Will create non-existing topics automatically.
+        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
+        # While it is not recommended in prod, it simplifies work in dev
+        'allow.auto.create.topics': 'true',
+        # We refresh the cluster state often as newly created topics in dev may not be detected
+        # fast enough. Fast enough means within reasonable time to provide decent user experience
+        # While it's only a one time thing for new topics, it can still be irritating to have to
+        # restart the process.
+        'topic.metadata.refresh.interval.ms': 5_000
+      }.freeze
+
+      private_constant :KAFKA_DEFAULTS, :DEV_DEFAULTS
 
       # Available settings
 
@@ -34,8 +47,6 @@ module Karafka
         setting :token, default: false
         # option entity [String] for whom we did issue the license
         setting :entity, default: ''
-        # option expires_on [Date] date when the license expires
-        setting :expires_on, default: Date.parse('2100-01-01')
       end
 
       # option client_id [String] kafka client_id - used to provide
@@ -136,8 +147,10 @@ module Karafka
 
           Contracts::Config.new.validate!(config.to_h)
 
-          # Check the license presence (if needed) and
-          Licenser.new.verify(config.license)
+          licenser = Licenser.new
+
+          # Tries to load our license gem and if present will try to load the correct license
+          licenser.prepare_and_verify(config.license)
 
           configure_components
 
@@ -149,13 +162,21 @@ module Karafka
         # Propagates the kafka setting defaults unless they are already present
         # This makes it easier to set some values that users usually don't change but still allows
         # them to overwrite the whole hash if they want to
-        # @param config [Dry::Configurable::Config] dry config of this producer
+        # @param config [Karafka::Core::Configurable::Node] config of this producer
         def merge_kafka_defaults!(config)
           KAFKA_DEFAULTS.each do |key, value|
             next if config.kafka.key?(key)
 
             config.kafka[key] = value
           end
+
+          return if Karafka::App.env.production?
+
+          DEV_DEFAULTS.each do |key, value|
+            next if config.kafka.key?(key)
+
+            config.kafka[key] = value
+          end
         end
 
         # Sets up all the components that are based on the user configuration

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.rc3'
+  VERSION = '2.0.0.rc6'
 end

data/lib/karafka.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 %w[
+  karafka-core
   delegate
   English
   rdkafka
@@ -12,8 +13,6 @@
   openssl
   base64
   date
-  dry/events/publisher
-  dry/monitor/notifications
   zeitwerk
 ].each(&method(:require))
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc3
+  version: 2.0.0.rc6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,36 +34,42 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-07-26 00:00:00.000000000 Z
+date: 2022-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: dry-monitor
+  name: karafka-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 2.0.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 2.0.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -84,7 +90,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.3
+        version: 2.4.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -94,7 +100,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.3
+        version: 2.4.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -192,6 +198,7 @@ files:
 - lib/karafka/instrumentation/logger.rb
 - lib/karafka/instrumentation/logger_listener.rb
 - lib/karafka/instrumentation/monitor.rb
+- lib/karafka/instrumentation/notifications.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
 - lib/karafka/instrumentation/vendors/datadog/listener.rb
@@ -277,7 +284,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"