karafka 2.0.0.alpha2 → 2.0.0.alpha3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 112496d14f74a8095df2b0b557e294140f422bbc8d6ea93f0709ac3b4ec017ec
-  data.tar.gz: a049427bd67d7887c0d5a352c724ecc56b156842859e9688068d36e25aeeb05d
+  metadata.gz: 4e715fa7f45c40a200fc2d281d765f5d42ac4f03ba6e3f529b86a0adc90bc1af
+  data.tar.gz: 50e2083063f1fd77ffc529395550e443c4166aa035c2d87716390059bec6fe32
 SHA512:
-  metadata.gz: d549e3c4bd31a2d011d453495ee6c3f3825ece067285834ef1bc5872cbc257f9695ff831eed7370c0668b33f4c297b1c3512fb068fecafdbd1a5d109d305e243
-  data.tar.gz: 2f7a48f6490e7ccef765c1697bdb54eb8e596246c853f8e30a85a134e50394c78c983b2ad4020d140da0a532aa1633cd425ba3af12a2d1e2f59fc9264288c3fb
+  metadata.gz: d225ba3d118716bcf1265e06d3d781aee7a3e9e87edaab791fdf1b240dc1565a0a460a3f99ca5b8687f6e63701ff2fee3d8ffe343dc18c5e1bfdde4af9deb852
+  data.tar.gz: 73084d0b5f3678f37b63debdcf944517e0f954bf0a21242caae3c807ced721afd26f5d6bdb864bcf5fe36bb2d098ca3dd167901a1c3ddc61b0ceb00f8a3a2d96

data/.github/workflows/ci.yml

@@ -113,6 +113,7 @@ jobs:
       - name: Install latest Bundler
         run: |
           gem install bundler --no-document
+          gem update --system --no-document
           bundle config set without 'tools benchmarks docs'
 
       - name: Bundle install
@@ -124,4 +125,4 @@ jobs:
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
-        run: bundle exec bin/integrations
+        run: bin/integrations

data/CHANGELOG.md

@@ -1,7 +1,14 @@
 # Karafka framework changelog
 
+## 2.0.0-alpha3 (2022-03-16)
+- Restore 'app.initialized' state and add notification on it
+- Fix the installation flow for Rails and add integration tests for this scenario
+- Add more integration tests covering some edge cases
+
 ## 2.0.0-alpha2 (2022-02-19)
 - Require `kafka` keys to be symbols
+- Added ActiveJob Pro adapter
+- Small updates to the license and docs
 
 ## 2.0.0-alpha1 (2022-01-30)
 - Change license to `LGPL-3.0`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.alpha2)
+    karafka (2.0.0.alpha3)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
@@ -71,7 +71,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    mini_portile2 (2.7.1)
+    mini_portile2 (2.8.0)
     minitest (5.15.0)
     rake (13.0.6)
     rdkafka (0.11.1)

data/bin/integrations

@@ -2,9 +2,15 @@
 
 # Runner to run integration specs in parallel
 
+# Part of integration specs run pristine without bundler.
+# If we would run bundle exec when running this code, bundler would inject its own context
+# into them, messing things up heavily
+raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
+
 require 'open3'
 require 'fileutils'
 require 'pathname'
+require 'tmpdir'
 
 ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
 
@@ -38,9 +44,7 @@ class Scenario
   # @param path [String] path to the scenarios file
   def initialize(path)
     @path = path
-    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(
-      "bundle exec ruby -r ./spec/integrations_helper.rb #{path}"
-    )
+    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
     @started_at = current_time
     # Last 1024 characters from stdout
     @stdout_tail = ''
@@ -102,6 +106,35 @@ class Scenario
 
   private
 
+  # Sets up a proper environment for a given spec to run and returns the run command
+  # @return [String] run command
+  def init_and_build_cmd
+    scenario_dir = File.dirname(@path)
+
+    # If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
+    # to run bundle install, etc in order to run it
+    if File.exist?(File.join(scenario_dir, 'Gemfile'))
+      # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
+      temp_dir = Dir.mktmpdir
+      file_name = File.basename(@path)
+
+      FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
+
+      <<~CMD
+        cd #{temp_dir} &&
+        KARAFKA_GEM_DIR=#{ROOT_PATH} \
+        BUNDLE_AUTO_INSTALL=true \
+        PRISTINE_MODE=true \
+        bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{file_name}
+      CMD
+    else
+      <<~CMD
+        KARAFKA_GEM_DIR=#{ROOT_PATH} \
+        bundle exec ruby -r ./spec/integrations_helper.rb #{@path}
+      CMD
+    end
+  end
+
   # @return [Float] current machine time
   def current_time
     Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -136,10 +169,13 @@ end
 # Load all the specs
 specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
 
-# If filter is provided, apply
-specs.delete_if { |name| !name.include?(ARGV[0]) } if ARGV[0]
+# If filters is provided, apply
+# Allows to provide several filters one after another and applies all of them
+ARGV.each do |filter|
+  specs.delete_if { |name| !name.include?(filter) }
+end
 
-raise ArgumentError, "No integration specs with filter: #{ARGV[0]}" if specs.empty?
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
 
 # Randomize order
 seed = (ENV['SEED'] || rand(0..10_000)).to_i

data/lib/karafka/cli/install.rb

@@ -14,6 +14,7 @@ module Karafka
         app/consumers
         config
         log
+        lib
       ].freeze
 
       # Where should we map proper files from templates

data/lib/karafka/instrumentation/monitor.rb

@@ -17,6 +17,7 @@ module Karafka
       #   complete list of all the events. Please use the #available_events on fully loaded
       #   Karafka system to determine all of the events you can use.
       BASE_EVENTS = %w[
+        app.initialized
         app.running
         app.stopping
         app.stopped

data/lib/karafka/railtie.rb

@@ -38,36 +38,62 @@ if rails
         end
       end
 
-      initializer 'karafka.configure_rails_initialization' do |app|
-        # Consumers should autoload by default in the Rails app so they are visible
-        app.config.autoload_paths += %w[app/consumers]
-
+      # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+      # server + will support code reloading with each fetched loop. We do it only for karafka
+      # based commands as Rails processes and console will have it enabled already
+      initializer 'karafka.configure_rails_logger' do
         # Make Karafka use Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
-        # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
-        # server + will support code reloading with each fetched loop. We do it only for karafka
-        # based commands as Rails processes and console will have it enabled already
-        if Rails.env.development? && ENV.key?('KARAFKA_CLI')
-          Rails.logger.extend(
-            ActiveSupport::Logger.broadcast(
-              ActiveSupport::Logger.new($stdout)
-            )
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+
+        Rails.logger.extend(
+          ActiveSupport::Logger.broadcast(
+            ActiveSupport::Logger.new($stdout)
           )
+        )
+      end
 
-          # We can have many listeners, but it does not matter in which we will reload the code as
-          # long as all the consumers will be re-created as Rails reload is thread-safe
-          ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-            # Reload code each time there is a change in the code
-            next unless Rails.application.reloaders.any?(&:updated?)
+      initializer 'karafka.configure_rails_auto_load_paths' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+      end
 
-            Rails.application.reloader.reload!
-          end
+      initializer 'karafka.configure_rails_code_reloader' do
+        # There are components that won't work with older Rails version, so we check it and
+        # provide a failover
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        next unless rails6plus
+
+        # We can have many listeners, but it does not matter in which we will reload the code
+        # as long as all the consumers will be re-created as Rails reload is thread-safe
+        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
+
+          Rails.application.reloader.reload!
         end
+      end
 
-        app.reloader.to_prepare do
-          # Load Karafka bot file, so it can be used in Rails server context
-          require Rails.root.join(Karafka.boot_file.to_s).to_s
+      initializer 'karafka.require_karafka_boot_file' do |app|
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
+
+        if rails6plus
+          app.reloader.to_prepare do
+            # Load Karafka boot file, so it can be used in Rails server context
+            require karafka_boot_file
+          end
+        else
+          # Load Karafka main setup for older Rails versions
+          app.config.after_initialize do
+            require karafka_boot_file
+          end
         end
       end
     end

data/lib/karafka/setup/config.rb

@@ -118,6 +118,8 @@ module Karafka
           Licenser.new.verify(config.license)
 
           configure_components
+
+          Karafka::App.initialized!
         end
 
         private

data/lib/karafka/status.rb

@@ -6,6 +6,7 @@ module Karafka
     # Available states and their transitions.
     STATES = {
       initializing: :initialize!,
+      initialized: :initialized!,
       running: :run!,
       stopping: :stop!,
       stopped: :stopped!

data/lib/karafka/templates/karafka.rb.erb

@@ -27,7 +27,7 @@ APP_LOADER.eager_load
 
 class KarafkaApp < Karafka::App
   setup do |config|
-    config.kafka = { 'bootstrap.servers' => '127.0.0.1:9092' }
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
     config.client_id = 'example_app'
 <% if rails? -%>
     # Recreate consumers with each batch. This will allow Rails code reload to work in the

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha2'
+  VERSION = '2.0.0.alpha3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha2
+  version: 2.0.0.alpha3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-02-19 00:00:00.000000000 Z
+date: 2022-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -151,7 +151,6 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
-- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
@@ -283,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/.github/FUNDING.yml

@@ -1,3 +0,0 @@
-# These are supported funding model platforms
-
-open_collective: karafka