karafka 1.4.12 → 2.2.10

data/lib/karafka/templates/karafka.rb.erb

@@ -1,26 +1,11 @@
 # frozen_string_literal: true
+<% unless rails? -%>
 
-<% if rails? -%>
-ENV['RAILS_ENV'] ||= 'development'
-ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
-require ::File.expand_path('../config/environment', __FILE__)
-Rails.application.eager_load!
-
-# This lines will make Karafka print to stdout like puma or unicorn
-if Rails.env.development?
-  Rails.logger.extend(
-    ActiveSupport::Logger.broadcast(
-      ActiveSupport::Logger.new($stdout)
-    )
-  )
-end
-<% else -%>
 # This file is auto-generated during the install process.
 # If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
 # command again or refer to the install templates available in the source codes
 
-ENV['RACK_ENV'] ||= 'development'
-ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
+ENV['KARAFKA_ENV'] ||= 'development'
 Bundler.require(:default, ENV['KARAFKA_ENV'])
 
 # Zeitwerk custom loader for loading the app components before the whole
@@ -31,9 +16,7 @@ APP_LOADER.enable_reloading
 %w[
   lib
   app/consumers
-  app/responders
-  app/workers
-].each(&APP_LOADER.method(:push_dir))
+].each { |dir| APP_LOADER.push_dir(dir) }
 
 APP_LOADER.setup
 APP_LOADER.eager_load
@@ -41,10 +24,12 @@ APP_LOADER.eager_load
 
 class KarafkaApp < Karafka::App
   setup do |config|
-    config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
     config.client_id = 'example_app'
 <% if rails? -%>
-    config.logger = Rails.logger
+    # Recreate consumers with each batch. This will allow Rails code reload to work in the
+    # development mode. Otherwise Karafka process would not be aware of code changes
+    config.consumer_persistence = !Rails.env.development?
 <% end -%>
   end
 
@@ -52,41 +37,34 @@ class KarafkaApp < Karafka::App
   # interested in logging events for certain environments. Since instrumentation
   # notifications add extra boilerplate, if you want to achieve max performance,
   # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(WaterDrop::Instrumentation::StdoutListener.new)
-  Karafka.monitor.subscribe(Karafka::Instrumentation::StdoutListener.new)
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
-  # Uncomment that in order to achieve code reload in development mode
-  # Be aware, that this might have some side-effects. Please refer to the wiki
-  # for more details on benefits and downsides of the code reload in the
-  # development mode
-  #
-  # Karafka.monitor.subscribe(
-  #   Karafka::CodeReloader.new(
-  #     <%= rails? ? '*Rails.application.reloaders' : 'APP_LOADER' %>
-  #   )
-  # )
-
-  consumer_groups.draw do
-    # topic :example do
-    #   consumer ExampleConsumer
-    # end
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(
+      # Log producer operations using the Karafka logger
+      Karafka.logger,
+      # If you set this to true, logs will contain each message details
+      # Please note, that this can be extensive
+      log_messages: false
+    )
+  )
 
-    # consumer_group :bigger_group do
-    #   topic :test do
-    #     consumer TestConsumer
-    #   end
-    #
-    #   topic :test2 do
-    #     consumer Test2Consumer
-    #   end
-    # end
+  routes.draw do
+<% if rails? -%>
+    # Uncomment this if you use Karafka with ActiveJob
+    # You need to define the topic per each queue name you use
+    # active_job_topic :default
+<% end -%>
+    topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
+      consumer ExampleConsumer
+    end
   end
 end
-
-Karafka.monitor.subscribe('app.initialized') do
-  # Put here all the things you want to do after the Karafka framework
-  # initialization
-end
-
-KarafkaApp.boot!

data/lib/karafka/time_trackers/base.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Time trackers module.
+  #
+  # Time trackers are used to track time in context of having a time poll (amount of time
+  # available for processing) or a pausing engine (pause for a time period).
+  module TimeTrackers
+    # Base class for all the time-trackers.
+    class Base
+      include ::Karafka::Core::Helpers::Time
+    end
+  end
+end

data/lib/karafka/time_trackers/pause.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Karafka
+  module TimeTrackers
+    # Handles Kafka topic partition pausing and resuming with exponential back-offs.
+    # Since expiring and pausing can happen from both consumer and listener, this needs to be
+    # thread-safe.
+    #
+    # @note We do not have to worry about performance implications of a mutex wrapping most of the
+    #   code here, as this is not a frequently used tracker. It is active only once per batch in
+    #   case of long-running-jobs and upon errors.
+    class Pause < Base
+      attr_reader :attempt, :current_timeout
+
+      # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
+      # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
+      #   exponentially on which we will stay
+      # @param exponential_backoff [Boolean] should we wait exponentially or with the same
+      #   timeout value
+      # @return [Karafka::TimeTrackers::Pause]
+      # @example
+      #   options = { timeout: 1000, max_timeout: 1000, exponential_backoff: false }
+      #   pause = Karafka::TimeTrackers::Pause.new(**options)
+      #   pause.expired? #=> true
+      #   pause.paused? #=> false
+      #   pause.pause
+      #   pause.increment
+      #   sleep(1.1)
+      #   pause.paused? #=> true
+      #   pause.expired? #=> true
+      #   pause.attempt #=> 1
+      #   pause.pause
+      #   pause.increment
+      #   pause.attempt #=> 2
+      #   pause.paused? #=> true
+      #   pause.expired? #=> false
+      #   pause.resume
+      #   pause.attempt #=> 2
+      #   pause.paused? #=> false
+      #   pause.reset
+      #   pause.attempt #=> 0
+      def initialize(timeout:, max_timeout:, exponential_backoff:)
+        @started_at = nil
+        @attempt = 0
+        @timeout = timeout
+        @current_timeout = timeout
+        @max_timeout = max_timeout
+        @exponential_backoff = exponential_backoff
+        @mutex = Mutex.new
+        super()
+      end
+
+      # Pauses the processing from now till the end of the interval (backoff or non-backoff)
+      # and records the attempt.
+      # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
+      # @note Providing this value can be useful when we explicitly want to pause for a certain
+      #   period of time, outside of any regular pausing logic
+      def pause(timeout = backoff_interval)
+        @mutex.synchronize do
+          @current_timeout = timeout
+          @started_at = monotonic_now
+          @ends_at = @started_at + timeout
+        end
+      end
+
+      # Increments the number of attempt by 1
+      def increment
+        @mutex.synchronize do
+          @attempt += 1
+        end
+      end
+
+      # Marks the pause as resumed.
+      def resume
+        @mutex.synchronize do
+          @started_at = nil
+          @ends_at = nil
+        end
+      end
+
+      # Expires the pause, so it can be considered expired
+      def expire
+        @mutex.synchronize do
+          @ends_at = nil
+        end
+      end
+
+      # @return [Boolean] are we paused from processing
+      def paused?
+        @mutex.synchronize do
+          !@started_at.nil?
+        end
+      end
+
+      # @return [Boolean] did the pause expire
+      def expired?
+        @mutex.synchronize do
+          @ends_at ? monotonic_now >= @ends_at : true
+        end
+      end
+
+      # Resets the pause attempt count.
+      def reset
+        @mutex.synchronize do
+          @attempt = 0
+        end
+      end
+
+      private
+
+      # Computers the exponential backoff
+      # @return [Integer] backoff in milliseconds
+      def backoff_interval
+        backoff_factor = @exponential_backoff ? 2**@attempt : 1
+
+        timeout = backoff_factor * @timeout
+
+        @max_timeout && timeout > @max_timeout ? @max_timeout : timeout
+      end
+    end
+  end
+end

data/lib/karafka/time_trackers/poll.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Karafka
+  module TimeTrackers
+    # Object used to keep track of time we've used running certain operations. Polling is
+    # running in a single thread, thus we do not have to worry about this being thread-safe.
+    #
+    # @example Keep track of sleeping and stop after 3 seconds of 0.1 sleep intervals
+    #   time_poll = Poll.new(3000)
+    #   time_poll.start
+    #
+    #   until time_poll.exceeded?
+    #     time_poll.start
+    #     puts "I have #{time_poll.remaining.to_i}ms remaining to sleep..."
+    #     sleep(0.1)
+    #     time_poll.checkpoint
+    #   end
+    class Poll < Base
+      attr_reader :remaining, :attempts
+
+      # @param total_time [Integer] amount of milliseconds before we exceed the given time limit
+      # @return [TimeTracker] time poll instance
+      def initialize(total_time)
+        @remaining = total_time
+        @attempts = 0
+        super()
+      end
+
+      # @return [Boolean] did we exceed the time limit
+      def exceeded?
+        @remaining <= 0
+      end
+
+      # Starts time tracking.
+      def start
+        @attempts += 1
+        @started_at = monotonic_now
+      end
+
+      # Stops time tracking of a given piece of code and updates the remaining time.
+      def checkpoint
+        @remaining -= (monotonic_now - @started_at)
+      end
+
+      # @return [Boolean] If anything went wrong, can we retry after a backoff period or not
+      #   (do we have enough time)
+      def retryable?
+        remaining > backoff_interval
+      end
+
+      # Sleeps for amount of time matching attempt, so we sleep more with each attempt in case of
+      #   a retry.
+      def backoff
+        # backoff should not be included in the remaining time computation, otherwise it runs
+        # shortly, never back-offing beyond a small number because of the sleep
+        @remaining += backoff_interval
+        # Sleep requires seconds not ms
+        sleep(backoff_interval / 1_000.0)
+      end
+
+      private
+
+      # @return [Integer] milliseconds of the backoff time
+      def backoff_interval
+        100 * attempts
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.4.12'
+  VERSION = '2.2.10'
 end

data/lib/karafka.rb

@@ -1,34 +1,50 @@
 # frozen_string_literal: true
 
 %w[
+  karafka-core
   delegate
   English
+  rdkafka
   waterdrop
-  kafka
-  envlogic
   json
-  thor
   forwardable
   fileutils
-  dry-configurable
-  dry-validation
-  dry/events/publisher
-  dry/inflector
-  dry/monitor/notifications
-  dry/core/constants
+  openssl
+  optparse
+  base64
+  date
+  singleton
+  digest
   zeitwerk
+  concurrent/atomic/atomic_fixnum
 ].each(&method(:require))
 
-# Karafka library
+# Karafka framework main namespace
 module Karafka
-  extend Envlogic
-
   class << self
+    # @return [Karafka::Env] env instance that allows us to check environment
+    def env
+      @env ||= Env.new
+    end
+
+    # @param environment [String, Symbol] new environment that we want to set
+    # @return [Karafka::Env] env instance
+    # @example Assign new environment to Karafka::App
+    #   Karafka::App.env = :production
+    def env=(environment)
+      env.replace(environment.to_s)
+    end
+
     # @return [Logger] logger that we want to use. Will use ::Karafka::Logger by default
     def logger
       @logger ||= App.config.logger
     end
 
+    # @return [WaterDrop::Producer] waterdrop messages producer
+    def producer
+      @producer ||= App.config.producer
+    end
+
     # @return [::Karafka::Monitor] monitor that we want to use
     def monitor
       @monitor ||= App.config.monitor
@@ -49,6 +65,24 @@ module Karafka
       Pathname.new(File.expand_path('karafka', __dir__))
     end
 
+    # @return [Boolean] true if there is a valid pro token present
+    def pro?
+      App.config.license.token != false
+    end
+
+    # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
+    #   the console invocation to Rails
+    def rails?
+      return @rails if instance_variable_defined?('@rails')
+
+      # Do not load Rails again if already loaded
+      Object.const_defined?('Rails::Railtie') || require('rails')
+
+      @rails = true
+    rescue LoadError
+      @rails = false
+    end
+
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
@@ -61,12 +95,52 @@ module Karafka
     def boot_file
       Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
     end
+
+    # We need to be able to overwrite both monitor and logger after the configuration in case they
+    # would be changed because those two (with defaults) can be used prior to the setup and their
+    # state change should be reflected in the updated setup
+    #
+    # This method refreshes the things that might have been altered by the configuration
+    def refresh!
+      config = ::Karafka::App.config
+
+      @logger = config.logger
+      @producer = config.producer
+      @monitor = config.monitor
+    end
   end
 end
 
-Zeitwerk::Loader
-  .for_gem
-  .tap(&:setup)
-  .tap(&:eager_load)
+loader = Zeitwerk::Loader.for_gem
+# Do not load Rails extensions by default, this will be handled by Railtie if they are needed
+loader.ignore(Karafka.gem_root.join('lib/active_job'))
+# Do not load CurrentAttributes components as they will be loaded if needed
+# @note We have to exclude both the .rb file as well as the whole directory so users can require
+# current attributes only when needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes'))
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes.rb'))
+# Do not load Railtie. It will load if after everything is ready, so we don't have to load any
+# Karafka components when we require this railtie. Railtie needs to be loaded last.
+loader.ignore(Karafka.gem_root.join('lib/karafka/railtie'))
+# Do not load pro components as they will be loaded if needed and allowed
+loader.ignore(Karafka.core_root.join('pro/'))
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore(Karafka.core_root.join('instrumentation/vendors'))
+loader.setup
+loader.eager_load
+
+# This will load features but since Pro are not loaded automatically, they will not be visible
+# nor included here
+::Karafka::Routing::Features::Base.load_all
+
+# We need to detect and require (not setup) Pro components during the gem load, because we need
+# to make pro components available in case anyone wants to use them as a base to their own
+# custom components. Otherwise inheritance would not work.
+Karafka::Licenser.detect do
+  require 'karafka/pro/loader'
+
+  Karafka::Pro::Loader.require_all
+end
 
-Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
+# Load railtie after everything else is ready so we know we can rely on it.
+require 'karafka/railtie'

data/renovate.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+      "config:base"
+  ],
+  "ignorePaths": [
+      "spec/integrations"
+  ]
+}