karafka 2.0.28 → 2.0.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c69f25e46161a6f20e57262b4eb67a852b2345e4cea05b4919bca43105fa2ca2
-  data.tar.gz: fb065461afccd8d2416bd509dfa6f04b47df07dd6b6ffcbcde9d266f35b71412
+  metadata.gz: 3a258a78f3df80b4cda4f315f56ef4f7fb48003f63d0d58140c189b01b3f4875
+  data.tar.gz: c267cb96bb8a972a2255f4a1298adde63678be4502b4ec8573af9c8a940435b4
 SHA512:
-  metadata.gz: 1819faf0d836a42cc6d05adde4cc896a31ec58d3b0de19f4f62d224fd16d1b3ebaa816afc45fc67025de579ac63c81cc09e18ffa4dacb8e2f05f636b21df24b5
-  data.tar.gz: 65437e05a791e8e0f86a17d08ab331ca52dfdadbb3979109b2358b6ba700ca63e6e956d0125784da0228adabf2aa92d18058f7f3440e72ec98b3659c9b127ba1
+  metadata.gz: beed9b0cbc83249d0bf227edb859142888d748dab6a972bf58c2d7e38ad578292544948bd9f2f25edbeebe6b37d1f0d20d47fc5e15d3d12420964447d414c54a
+  data.tar.gz: 7da728f75e5540d339887d2fb3bee4787016bdcd16a56f29b4f9c0b3e9f4ff45f1d6f0e843d25bf3e7a31e05ea3a320e174a005a149f76156618dc70327e71de

data/CHANGELOG.md

@@ -1,8 +1,15 @@
 # Karafka framework changelog
 
+## 2.0.29 (2023-01-30)
+- [Improvement] Make sure, that the `Karafka#producer` instance has the `LoggerListener` enabled in the install template, so Karafka by default prints both consumer and producer info.
+- [Improvement] Extract the code loading capabilities of Karafka console from the executable, so web can use it to provide CLI commands.
+- [Fix] Fix for: running karafka console results in NameError with Rails (#1280)
+- [Fix] Make sure, that the `caller` for async errors is being published.
+- [Change] Make sure that WaterDrop `2.4.10` or higher is used with this release to support Web-UI.
+
 ## 2.0.28 (2023-01-25)
 - **[Feature]** Provide the ability to use Dead Letter Queue with Virtual Partitions.
-- [Improvement] Collapse Virtual Partitions upon retryable error to a single partition. This allows dead letter queue to operate and mitigate issues arising from work virtualization. This removes uncertainties upon errors that can be retried and processed. Affects given topic partition virtualization only for multi-topic and mulit-partition parallelization. It also minimizes potential "flickering" where given data set has potentially many corrupted messages. The collapse will last until all the messages from the collective corrupted batch are processed. After that, virtualization will resume.
+- [Improvement] Collapse Virtual Partitions upon retryable error to a single partition. This allows dead letter queue to operate and mitigate issues arising from work virtualization. This removes uncertainties upon errors that can be retried and processed. Affects given topic partition virtualization only for multi-topic and multi-partition parallelization. It also minimizes potential "flickering" where given data set has potentially many corrupted messages. The collapse will last until all the messages from the collective corrupted batch are processed. After that, virtualization will resume.
 - [Improvement] Introduce `#collapsed?` consumer method available for consumers using Virtual Partitions.
 - [Improvement] Allow for customization of DLQ dispatched message details in Pro (#1266) via the `#enhance_dlq_message` consumer method.
 - [Improvement] Include `original_consumer_group` in the DLQ dispatched messages in Pro.

data/Gemfile.lock

@@ -1,25 +1,25 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.28)
+    karafka (2.0.29)
       karafka-core (>= 2.0.9, < 3.0.0)
       thor (>= 0.20)
-      waterdrop (>= 2.4.9, < 3.0.0)
+      waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.4.1)
-      activesupport (= 7.0.4.1)
+    activejob (7.0.4.2)
+      activesupport (= 7.0.4.2)
       globalid (>= 0.3.6)
-    activesupport (7.0.4.1)
+    activesupport (7.0.4.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
     factory_bot (6.2.1)
@@ -61,12 +61,13 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.9)
+    waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.6)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -78,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.2
+   2.4.5

data/README.md

@@ -58,7 +58,8 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-bundle add karafka
+# Make sure to install Karafka 2.0
+bundle add karafka --version ">= 2.0.28"
 
 bundle exec karafka install
 ```
@@ -86,7 +87,7 @@ I also sell Karafka Pro subscriptions. It includes a commercial-friendly license
 
 **20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
 
-Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io) for more details.
+Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io/#become-pro) for more details.
 
 ## Support
 

data/bin/karafka

@@ -6,24 +6,6 @@ require 'karafka'
 # our bin/karafka cli
 ENV['KARAFKA_CLI'] = 'true'
 
-# If there is a boot file, we need to require it as we expect it to contain
-# Karafka app setup, routes, etc
-if File.exist?(Karafka.boot_file)
-  rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
-
-  # Load Rails environment file that starts Rails, so we can reference consumers and other things
-  # from `karafka.rb` file. This will work only for Rails, for non-rails a manual setup is needed
-  require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
-
-  require Karafka.boot_file.to_s
-else
-  # However when it is unavailable, we still want to be able to run help command
-  # and install command as they don't require configured app itself to run
-  raise(
-    Karafka::Errors::MissingBootFileError,
-    Karafka.boot_file
-  ) unless %w[-h install].include?(ARGV[0])
-end
-
+Karafka::Cli::Base.load
 Karafka::Cli.prepare
 Karafka::Cli.start

data/bin/verify_license_integrity

@@ -11,6 +11,8 @@ set -e
 if [ "$MODE" != "after" ]; then
   # Check the remote license prior to bundle installing
   curl \
+    --retry 5 \
+    --retry-delay 1 \
     --fail \
     -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
     https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \

data/karafka.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'karafka-core', '>= 2.0.9', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.9', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.10', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/cli/base.rb

@@ -34,6 +34,26 @@ module Karafka
       end
 
       class << self
+        # Loads proper environment with what is needed to run the CLI
+        def load
+          # If there is a boot file, we need to require it as we expect it to contain
+          # Karafka app setup, routes, etc
+          if File.exist?(::Karafka.boot_file)
+            rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
+
+            # Load Rails environment file that starts Rails, so we can reference consumers and
+            # other things from `karafka.rb` file. This will work only for Rails, for non-rails
+            # a manual setup is needed
+            require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
+
+            require Karafka.boot_file.to_s
+          # However when it is unavailable, we still want to be able to run help command
+          # and install command as they don't require configured app itself to run
+          elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
+            raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
+          end
+        end
+
         # Allows to set options for Thor cli
         # @see https://github.com/erikhuda/thor
         # @param option Single option details

data/lib/karafka/cli/console.rb

@@ -9,22 +9,27 @@ module Karafka
       option aliases: 'c'
 
       class << self
-        # @return [String] Console executing command
+        # @return [String] Console executing command for non-Rails setup
         # @example
         #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-        def command
-          envs = [
-            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-            'KARAFKA_CONSOLE=true'
-          ]
-          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        def console
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc' bundle exec irb -r #{Karafka.boot_file}"
+        end
+
+        # @return [String] Console executing command for Rails setup
+        # @note In case of Rails, it has its own console, hence we can just defer to it
+        def rails_console
+          'bundle exec rails console'
         end
       end
 
       # Start the Karafka console
       def call
         cli.info
-        exec self.class.command
+
+        command = ::Karafka.rails? ? self.class.rails_console : self.class.console
+
+        exec "KARAFKA_CONSOLE=true #{command}"
       end
     end
   end

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -28,6 +28,7 @@ module Karafka
 
           @monitor.instrument(
             'error.occurred',
+            caller: self,
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,
             type: 'librdkafka.error',

data/lib/karafka/railtie.rb

@@ -2,14 +2,7 @@
 
 # This file contains Railtie for auto-configuration
 
-rails = false
-
-begin
-  # Do not load Rails again if already loaded
-  Object.const_defined?('Rails::Railtie') || require('rails')
-
-  rails = true
-rescue LoadError
+unless Karafka.rails?
   # Without defining this in any way, Zeitwerk ain't happy so we do it that way
   module Karafka
     class Railtie
@@ -17,7 +10,7 @@ rescue LoadError
   end
 end
 
-if rails
+if Karafka.rails?
   # Load ActiveJob adapter
   require 'active_job/karafka'
 

data/lib/karafka/templates/karafka.rb.erb

@@ -40,6 +40,12 @@ class KarafkaApp < Karafka::App
   Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
+  )
+
   routes.draw do
 <% if rails? -%>
     # Uncomment this if you use Karafka with ActiveJob

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.28'
+  VERSION = '2.0.29'
 end

data/lib/karafka.rb

@@ -70,6 +70,19 @@ module Karafka
       App.config.license.token != false
     end
 
+    # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
+    #   the console invocation to Rails
+    def rails?
+      return @rails if instance_variable_defined?('@rails')
+
+      # Do not load Rails again if already loaded
+      Object.const_defined?('Rails::Railtie') || require('rails')
+
+      @rails = true
+    rescue LoadError
+      @rails = false
+    end
+
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.28
+  version: 2.0.29
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-01-25 00:00:00.000000000 Z
+date: 2023-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.9
+        version: 2.4.10
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.9
+        version: 2.4.10
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0