karafka 2.0.0.alpha2 → 2.0.0.alpha5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 112496d14f74a8095df2b0b557e294140f422bbc8d6ea93f0709ac3b4ec017ec
-  data.tar.gz: a049427bd67d7887c0d5a352c724ecc56b156842859e9688068d36e25aeeb05d
+  metadata.gz: 062ee1f4c49d482daa48aa1af06aae08b83fb879ecfb4a6a14d5b2b34ed2975a
+  data.tar.gz: 28b92b8b5cea506641e339d59f53122fb18deb1637c20a2b34038ae2000b6f17
 SHA512:
-  metadata.gz: d549e3c4bd31a2d011d453495ee6c3f3825ece067285834ef1bc5872cbc257f9695ff831eed7370c0668b33f4c297b1c3512fb068fecafdbd1a5d109d305e243
-  data.tar.gz: 2f7a48f6490e7ccef765c1697bdb54eb8e596246c853f8e30a85a134e50394c78c983b2ad4020d140da0a532aa1633cd425ba3af12a2d1e2f59fc9264288c3fb
+  metadata.gz: '0497e94e2aa16ee20ded58e17313c9259bacb6f3fa91259e30f0cb0560b58a1eecb11447fb150e09cd659bb24d48e0cc732dc3fc7cc585a2aefe980df2e5b3f1'
+  data.tar.gz: 280407edd6298a7e62f970d2f9a2d6d1ff6eba4737ea4369eba20c4a83b9d00a0efa24ef36ceabd9def060839039fcf925c341f520efa2ab68532d484482f4fd

data/.github/workflows/ci.yml

@@ -113,6 +113,7 @@ jobs:
       - name: Install latest Bundler
         run: |
           gem install bundler --no-document
+          gem update --system --no-document
           bundle config set without 'tools benchmarks docs'
 
       - name: Bundle install
@@ -124,4 +125,4 @@ jobs:
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
-        run: bundle exec bin/integrations
+        run: bin/integrations

data/CHANGELOG.md

@@ -1,7 +1,20 @@
 # Karafka framework changelog
 
+## 2.0.0-alpha5 (2022-04-03)
+- Rename StdoutListener to LoggerListener (#811)
+
+## 2.0.0-alpha4 (2022-03-20)
+- Rails support without ActiveJob queue adapter usage (#805)
+
+## 2.0.0-alpha3 (2022-03-16)
+- Restore 'app.initialized' state and add notification on it
+- Fix the installation flow for Rails and add integration tests for this scenario
+- Add more integration tests covering some edge cases
+
 ## 2.0.0-alpha2 (2022-02-19)
 - Require `kafka` keys to be symbols
+- Added ActiveJob Pro adapter
+- Small updates to the license and docs
 
 ## 2.0.0-alpha1 (2022-01-30)
 - Change license to `LGPL-3.0`

data/Gemfile.lock

@@ -1,28 +1,28 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.alpha2)
+    karafka (2.0.0.alpha5)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
       rdkafka (>= 0.10)
       thor (>= 0.20)
-      waterdrop (>= 2.2.0, < 3.0.0)
+      waterdrop (>= 2.3.0, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.2.2)
-      activesupport (= 7.0.2.2)
+    activejob (7.0.2.3)
+      activesupport (= 7.0.2.3)
       globalid (>= 0.3.6)
-    activesupport (7.0.2.2)
+    activesupport (7.0.2.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     diff-lcs (1.5.0)
     docile (1.4.0)
     dry-configurable (0.14.0)
@@ -64,14 +64,14 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
       dry-schema (~> 1.9, >= 1.9.1)
-    factory_bot (6.2.0)
+    factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
     globalid (1.0.0)
       activesupport (>= 5.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    mini_portile2 (2.7.1)
+    mini_portile2 (2.8.0)
     minitest (5.15.0)
     rake (13.0.6)
     rdkafka (0.11.1)
@@ -87,7 +87,7 @@ GEM
     rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
     rspec-support (3.11.0)
@@ -100,7 +100,7 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.2.0)
+    waterdrop (2.3.0)
       concurrent-ruby (>= 1.1)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
@@ -121,4 +121,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.7
+   2.3.10

data/bin/integrations

@@ -2,9 +2,15 @@
 
 # Runner to run integration specs in parallel
 
+# Part of integration specs run pristine without bundler.
+# If we would run bundle exec when running this code, bundler would inject its own context
+# into them, messing things up heavily
+raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
+
 require 'open3'
 require 'fileutils'
 require 'pathname'
+require 'tmpdir'
 
 ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
 
@@ -19,7 +25,7 @@ CONCURRENCY = 4
 class Scenario
   # How long a scenario can run before we kill it
   # This is a fail-safe just in case something would hang
-  MAX_RUN_TIME = 60 * 5
+  MAX_RUN_TIME = 60 * 2
 
   # There are rare cases where Karafka may force shutdown for some of the integration cases
   # This includes exactly those
@@ -38,9 +44,7 @@ class Scenario
   # @param path [String] path to the scenarios file
   def initialize(path)
     @path = path
-    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(
-      "bundle exec ruby -r ./spec/integrations_helper.rb #{path}"
-    )
+    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
     @started_at = current_time
     # Last 1024 characters from stdout
     @stdout_tail = ''
@@ -102,6 +106,35 @@ class Scenario
 
   private
 
+  # Sets up a proper environment for a given spec to run and returns the run command
+  # @return [String] run command
+  def init_and_build_cmd
+    scenario_dir = File.dirname(@path)
+
+    # If there is a Gemfile in a scenario directory, it means it is a pristine spec and we need
+    # to run bundle install, etc in order to run it
+    if File.exist?(File.join(scenario_dir, 'Gemfile'))
+      # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
+      temp_dir = Dir.mktmpdir
+      file_name = File.basename(@path)
+
+      FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
+
+      <<~CMD
+        cd #{temp_dir} &&
+        KARAFKA_GEM_DIR=#{ROOT_PATH} \
+        BUNDLE_AUTO_INSTALL=true \
+        PRISTINE_MODE=true \
+        bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{file_name}
+      CMD
+    else
+      <<~CMD
+        KARAFKA_GEM_DIR=#{ROOT_PATH} \
+        bundle exec ruby -r ./spec/integrations_helper.rb #{@path}
+      CMD
+    end
+  end
+
   # @return [Float] current machine time
   def current_time
     Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -136,10 +169,13 @@ end
 # Load all the specs
 specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
 
-# If filter is provided, apply
-specs.delete_if { |name| !name.include?(ARGV[0]) } if ARGV[0]
+# If filters is provided, apply
+# Allows to provide several filters one after another and applies all of them
+ARGV.each do |filter|
+  specs.delete_if { |name| !name.include?(filter) }
+end
 
-raise ArgumentError, "No integration specs with filter: #{ARGV[0]}" if specs.empty?
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
 
 # Randomize order
 seed = (ENV['SEED'] || rand(0..10_000)).to_i

data/bin/stress

@@ -9,5 +9,5 @@ set -e
 while :
 do
   reset
-  bundle exec bin/integrations $1
+  bin/integrations $1
 done

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'dry-validation', '~> 1.7'
   spec.add_dependency 'rdkafka', '>= 0.10'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.2.0', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.3.0', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.6.0'

data/lib/active_job/karafka.rb

@@ -1,20 +1,21 @@
 # frozen_string_literal: true
 
-require 'active_job'
-require 'active_job/queue_adapters'
-require 'active_job/queue_adapters/karafka_adapter'
+begin
+  require 'active_job'
+  require_relative 'queue_adapters/karafka_adapter'
 
-module ActiveJob
-  # Namespace for usage simplification outside of Rails where Railtie will not kick in.
-  # That way a require 'active_job/karafka' should be enough to use it
-  module Karafka
+  module ActiveJob
+    # Namespace for usage simplification outside of Rails where Railtie will not kick in.
+    # That way a require 'active_job/karafka' should be enough to use it
+    module Karafka
+    end
   end
-end
-
-# We extend routing builder by adding a simple wrapper for easier jobs topics defining
-# This needs to be extended here as it is going to be used in karafka routes, hence doing that in
-# the railtie initializer would be too late
-::Karafka::Routing::Builder.include ::Karafka::ActiveJob::RoutingExtensions
-::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::RoutingExtensions
 
-# We extend ActiveJob stuff in the railtie
+  # We extend routing builder by adding a simple wrapper for easier jobs topics defining
+  # This needs to be extended here as it is going to be used in karafka routes, hence doing that in
+  # the railtie initializer would be too late
+  ::Karafka::Routing::Builder.include ::Karafka::ActiveJob::RoutingExtensions
+  ::Karafka::Routing::Proxy.include ::Karafka::ActiveJob::RoutingExtensions
+rescue LoadError
+  # We extend ActiveJob stuff in the railtie
+end

data/lib/karafka/cli/install.rb

@@ -14,6 +14,7 @@ module Karafka
         app/consumers
         config
         log
+        lib
       ].freeze
 
       # Where should we map proper files from templates

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -10,6 +10,7 @@ module Karafka
         required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
         required(:kafka).filled
         required(:max_messages) { int? & gteq?(1) }
+        required(:initial_offset).filled(included_in?: %w[earliest latest])
         required(:max_wait_time).filled { int? & gteq?(10) }
         required(:manual_offset_management).filled(:bool?)
         required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)

data/lib/karafka/instrumentation/{stdout_listener.rb → logger_listener.rb}

@@ -4,7 +4,7 @@ module Karafka
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
     # It can be removed/replaced or anything without any harm to the Karafka app flow.
-    class StdoutListener
+    class LoggerListener
       # Log levels that we use in this particular listener
       USED_LOG_LEVELS = %i[
         debug

data/lib/karafka/instrumentation/monitor.rb

@@ -17,6 +17,7 @@ module Karafka
       #   complete list of all the events. Please use the #available_events on fully loaded
       #   Karafka system to determine all of the events you can use.
       BASE_EVENTS = %w[
+        app.initialized
         app.running
         app.stopping
         app.stopped

data/lib/karafka/railtie.rb

@@ -38,36 +38,62 @@ if rails
         end
       end
 
-      initializer 'karafka.configure_rails_initialization' do |app|
-        # Consumers should autoload by default in the Rails app so they are visible
-        app.config.autoload_paths += %w[app/consumers]
-
+      # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
+      # server + will support code reloading with each fetched loop. We do it only for karafka
+      # based commands as Rails processes and console will have it enabled already
+      initializer 'karafka.configure_rails_logger' do
         # Make Karafka use Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
-        # This lines will make Karafka print to stdout like puma or unicorn when we run karafka
-        # server + will support code reloading with each fetched loop. We do it only for karafka
-        # based commands as Rails processes and console will have it enabled already
-        if Rails.env.development? && ENV.key?('KARAFKA_CLI')
-          Rails.logger.extend(
-            ActiveSupport::Logger.broadcast(
-              ActiveSupport::Logger.new($stdout)
-            )
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+
+        Rails.logger.extend(
+          ActiveSupport::Logger.broadcast(
+            ActiveSupport::Logger.new($stdout)
           )
+        )
+      end
 
-          # We can have many listeners, but it does not matter in which we will reload the code as
-          # long as all the consumers will be re-created as Rails reload is thread-safe
-          ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-            # Reload code each time there is a change in the code
-            next unless Rails.application.reloaders.any?(&:updated?)
+      initializer 'karafka.configure_rails_auto_load_paths' do |app|
+        # Consumers should autoload by default in the Rails app so they are visible
+        app.config.autoload_paths += %w[app/consumers]
+      end
 
-            Rails.application.reloader.reload!
-          end
+      initializer 'karafka.configure_rails_code_reloader' do
+        # There are components that won't work with older Rails version, so we check it and
+        # provide a failover
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        next unless Rails.env.development?
+        next unless ENV.key?('KARAFKA_CLI')
+        next unless rails6plus
+
+        # We can have many listeners, but it does not matter in which we will reload the code
+        # as long as all the consumers will be re-created as Rails reload is thread-safe
+        ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
+
+          Rails.application.reloader.reload!
         end
+      end
 
-        app.reloader.to_prepare do
-          # Load Karafka bot file, so it can be used in Rails server context
-          require Rails.root.join(Karafka.boot_file.to_s).to_s
+      initializer 'karafka.require_karafka_boot_file' do |app|
+        rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
+
+        karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
+
+        if rails6plus
+          app.reloader.to_prepare do
+            # Load Karafka boot file, so it can be used in Rails server context
+            require karafka_boot_file
+          end
+        else
+          # Load Karafka main setup for older Rails versions
+          app.config.after_initialize do
+            require karafka_boot_file
+          end
         end
       end
     end

data/lib/karafka/routing/subscription_group.rb

@@ -41,7 +41,7 @@ module Karafka
 
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id
-        kafka[:'auto.offset.reset'] ||= 'earliest'
+        kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
         kafka[:'enable.auto.offset.store'] = 'false'

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -18,6 +18,7 @@ module Karafka
         kafka
         max_messages
         max_wait_time
+        initial_offset
       ].freeze
 
       private_constant :DISTRIBUTION_KEYS

data/lib/karafka/routing/topic.rb

@@ -16,6 +16,7 @@ module Karafka
         manual_offset_management
         max_messages
         max_wait_time
+        initial_offset
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES

data/lib/karafka/setup/config.rb

@@ -54,6 +54,9 @@ module Karafka
       setting :consumer_persistence, default: true
       # Default deserializer for converting incoming data into ruby objects
       setting :deserializer, default: Karafka::Serialization::Json::Deserializer.new
+      # option [String] should we start with the earliest possible offset or latest
+      # This will set the `auto.offset.reset` value unless present in the kafka scope
+      setting :initial_offset, default: 'earliest'
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
@@ -118,6 +121,8 @@ module Karafka
           Licenser.new.verify(config.license)
 
           configure_components
+
+          Karafka::App.initialized!
         end
 
         private

data/lib/karafka/status.rb

@@ -6,6 +6,7 @@ module Karafka
     # Available states and their transitions.
     STATES = {
       initializing: :initialize!,
+      initialized: :initialized!,
       running: :run!,
       stopping: :stop!,
       stopped: :stopped!

data/lib/karafka/templates/karafka.rb.erb

@@ -27,7 +27,7 @@ APP_LOADER.eager_load
 
 class KarafkaApp < Karafka::App
   setup do |config|
-    config.kafka = { 'bootstrap.servers' => '127.0.0.1:9092' }
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
     config.client_id = 'example_app'
 <% if rails? -%>
     # Recreate consumers with each batch. This will allow Rails code reload to work in the
@@ -40,7 +40,7 @@ class KarafkaApp < Karafka::App
   # interested in logging events for certain environments. Since instrumentation
   # notifications add extra boilerplate, if you want to achieve max performance,
   # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(Karafka::Instrumentation::StdoutListener.new)
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
   routes.draw do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha2'
+  VERSION = '2.0.0.alpha5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha2
+  version: 2.0.0.alpha5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-02-19 00:00:00.000000000 Z
+date: 2022-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -112,7 +112,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -151,7 +151,6 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
-- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
@@ -212,9 +211,9 @@ files:
 - lib/karafka/instrumentation/callbacks/error.rb
 - lib/karafka/instrumentation/callbacks/statistics.rb
 - lib/karafka/instrumentation/logger.rb
+- lib/karafka/instrumentation/logger_listener.rb
 - lib/karafka/instrumentation/monitor.rb
 - lib/karafka/instrumentation/proctitle_listener.rb
-- lib/karafka/instrumentation/stdout_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -283,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/.github/FUNDING.yml

@@ -1,3 +0,0 @@
-# These are supported funding model platforms
-
-open_collective: karafka