karafka 1.4.13 → 2.0.0

data/lib/karafka/templates/karafka.rb.erb

@@ -1,19 +1,7 @@
 # frozen_string_literal: true
 
 <% if rails? -%>
-ENV['RAILS_ENV'] ||= 'development'
-ENV['KARAFKA_ENV'] = ENV['RAILS_ENV']
 require ::File.expand_path('../config/environment', __FILE__)
-Rails.application.eager_load!
-
-# This lines will make Karafka print to stdout like puma or unicorn
-if Rails.env.development?
-  Rails.logger.extend(
-    ActiveSupport::Logger.broadcast(
-      ActiveSupport::Logger.new($stdout)
-    )
-  )
-end
 <% else -%>
 # This file is auto-generated during the install process.
 # If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
@@ -31,9 +19,7 @@ APP_LOADER.enable_reloading
 %w[
   lib
   app/consumers
-  app/responders
-  app/workers
-].each(&APP_LOADER.method(:push_dir))
+].each { |dir| APP_LOADER.push_dir(dir) }
 
 APP_LOADER.setup
 APP_LOADER.eager_load
@@ -41,10 +27,12 @@ APP_LOADER.eager_load
 
 class KarafkaApp < Karafka::App
   setup do |config|
-    config.kafka.seed_brokers = %w[kafka://127.0.0.1:9092]
+    config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
     config.client_id = 'example_app'
 <% if rails? -%>
-    config.logger = Rails.logger
+    # Recreate consumers with each batch. This will allow Rails code reload to work in the
+    # development mode. Otherwise Karafka process would not be aware of code changes
+    config.consumer_persistence = !Rails.env.development?
 <% end -%>
   end
 
@@ -52,41 +40,17 @@ class KarafkaApp < Karafka::App
   # interested in logging events for certain environments. Since instrumentation
   # notifications add extra boilerplate, if you want to achieve max performance,
   # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(WaterDrop::Instrumentation::StdoutListener.new)
-  Karafka.monitor.subscribe(Karafka::Instrumentation::StdoutListener.new)
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
-  # Uncomment that in order to achieve code reload in development mode
-  # Be aware, that this might have some side-effects. Please refer to the wiki
-  # for more details on benefits and downsides of the code reload in the
-  # development mode
-  #
-  # Karafka.monitor.subscribe(
-  #   Karafka::CodeReloader.new(
-  #     <%= rails? ? '*Rails.application.reloaders' : 'APP_LOADER' %>
-  #   )
-  # )
-
-  consumer_groups.draw do
-    # topic :example do
-    #   consumer ExampleConsumer
-    # end
-
-    # consumer_group :bigger_group do
-    #   topic :test do
-    #     consumer TestConsumer
-    #   end
-    #
-    #   topic :test2 do
-    #     consumer Test2Consumer
-    #   end
-    # end
+  routes.draw do
+<% if rails? -%>
+    # Uncomment this if you use Karafka with ActiveJob
+    # You ned to define the topic per each queue name you use
+    # active_job_topic :default
+<% end -%>
+    topic :example do
+      consumer ExampleConsumer
+    end
   end
 end
-
-Karafka.monitor.subscribe('app.initialized') do
-  # Put here all the things you want to do after the Karafka framework
-  # initialization
-end
-
-KarafkaApp.boot!

data/lib/karafka/time_trackers/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Time trackers module.
+  #
+  # Time trackers are used to track time in context of having a time poll (amount of time
+  # available for processing) or a pausing engine (pause for a time period).
+  module TimeTrackers
+    # Base class for all the time-trackers.
+    class Base
+      private
+
+      # @return [Float] current time in milliseconds
+      def now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000
+      end
+    end
+  end
+end

data/lib/karafka/time_trackers/pause.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Karafka
+  module TimeTrackers
+    # Handles Kafka topic partition pausing and resuming with exponential back-offs.
+    class Pause < Base
+      attr_reader :count
+
+      # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
+      # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
+      #   exponentially on which we will stay
+      # @param exponential_backoff [Boolean] should we wait exponentially or with the same
+      #   timeout value
+      # @return [Karafka::TimeTrackers::Pause]
+      # @example
+      #   pause = Karafka::TimeTrackers::Pause.new(timeout: 1000)
+      #   pause.expired? #=> true
+      #   pause.paused? #=> false
+      #   pause.pause
+      #   sleep(1.1)
+      #   pause.paused? #=> true
+      #   pause.expired? #=> true
+      #   pause.count #=> 1
+      #   pause.pause
+      #   pause.count #=> 1
+      #   pause.paused? #=> true
+      #   pause.expired? #=> false
+      #   pause.resume
+      #   pause.count #=> 2
+      #   pause.paused? #=> false
+      #   pause.reset
+      #   pause.count #=> 0
+      def initialize(timeout:, max_timeout:, exponential_backoff:)
+        @started_at = nil
+        @count = 0
+        @timeout = timeout
+        @max_timeout = max_timeout
+        @exponential_backoff = exponential_backoff
+        super()
+      end
+
+      # Pauses the processing from now till the end of the interval (backoff or non-backoff)
+      # and records the count.
+      # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
+      # @note Providing this value can be useful when we explicitly want to pause for a certain
+      #   period of time, outside of any regular pausing logic
+      def pause(timeout = backoff_interval)
+        @started_at = now
+        @ends_at = @started_at + timeout
+        @count += 1
+      end
+
+      # Marks the pause as resumed.
+      def resume
+        @started_at = nil
+        @ends_at = nil
+      end
+
+      # Expires the pause, so it can be considered expired
+      def expire
+        @ends_at = nil
+      end
+
+      # @return [Boolean] are we paused from processing
+      def paused?
+        !@started_at.nil?
+      end
+
+      # @return [Boolean] did the pause expire
+      def expired?
+        @ends_at ? now >= @ends_at : true
+      end
+
+      # Resets the pause counter.
+      def reset
+        @count = 0
+      end
+
+      private
+
+      # Computers the exponential backoff
+      # @return [Integer] backoff in milliseconds
+      def backoff_interval
+        backoff_factor = @exponential_backoff ? 2**@count : 1
+
+        timeout = backoff_factor * @timeout
+
+        @max_timeout && timeout > @max_timeout ? @max_timeout : timeout
+      end
+    end
+  end
+end

data/lib/karafka/time_trackers/poll.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module TimeTrackers
+    # Object used to keep track of time we've used running certain operations.
+    #
+    # @example Keep track of sleeping and stop after 3 seconds of 0.1 sleep intervals
+    #   time_poll = Poll.new(3000)
+    #   time_poll.start
+    #
+    #   until time_poll.exceeded?
+    #     time_poll.start
+    #     puts "I have #{time_poll.remaining.to_i}ms remaining to sleep..."
+    #     sleep(0.1)
+    #     time_poll.checkpoint
+    #   end
+    class Poll < Base
+      attr_reader :remaining, :attempts
+
+      # @param total_time [Integer] amount of milliseconds before we exceed the given time limit
+      # @return [TimeTracker] time poll instance
+      def initialize(total_time)
+        @remaining = total_time
+        @attempts = 0
+        super()
+      end
+
+      # @return [Boolean] did we exceed the time limit
+      def exceeded?
+        @remaining <= 0
+      end
+
+      # Starts time tracking.
+      def start
+        @attempts += 1
+        @started_at = now
+      end
+
+      # Stops time tracking of a given piece of code and updates the remaining time.
+      def checkpoint
+        @remaining -= (now - @started_at)
+      end
+
+      # @return [Boolean] If anything went wrong, can we retry after a backoff period or not
+      #   (do we have enough time)
+      def retryable?
+        remaining > backoff_interval
+      end
+
+      # Sleeps for amount of time matching attempt, so we sleep more with each attempt in case of
+      #   a retry.
+      def backoff
+        # Sleep requires seconds not ms
+        sleep(backoff_interval / 1_000.0)
+      end
+
+      private
+
+      # @return [Integer] milliseconds of the backoff time
+      def backoff_interval
+        100 * attempts
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '1.4.13'
+  VERSION = '2.0.0'
 end

data/lib/karafka.rb

@@ -1,34 +1,47 @@
 # frozen_string_literal: true
 
 %w[
+  karafka-core
   delegate
   English
+  rdkafka
   waterdrop
-  kafka
-  envlogic
   json
   thor
   forwardable
   fileutils
-  dry-configurable
-  dry-validation
-  dry/events/publisher
-  dry/inflector
-  dry/monitor/notifications
-  dry/core/constants
+  openssl
+  base64
+  date
   zeitwerk
 ].each(&method(:require))
 
-# Karafka library
+# Karafka framework main namespace
 module Karafka
-  extend Envlogic
-
   class << self
+    # @return [Karafka::Env] env instance that allows us to check environment
+    def env
+      @env ||= Env.new
+    end
+
+    # @param environment [String, Symbol] new environment that we want to set
+    # @return [Karafka::Env] env instance
+    # @example Assign new environment to Karafka::App
+    #   Karafka::App.env = :production
+    def env=(environment)
+      env.replace(environment.to_s)
+    end
+
     # @return [Logger] logger that we want to use. Will use ::Karafka::Logger by default
     def logger
       @logger ||= App.config.logger
     end
 
+    # @return [WaterDrop::Producer] waterdrop messages producer
+    def producer
+      @producer ||= App.config.producer
+    end
+
     # @return [::Karafka::Monitor] monitor that we want to use
     def monitor
       @monitor ||= App.config.monitor
@@ -49,6 +62,11 @@ module Karafka
       Pathname.new(File.expand_path('karafka', __dir__))
     end
 
+    # @return [Boolean] true if there is a valid pro token present
+    def pro?
+      App.config.license.token != false
+    end
+
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
@@ -64,9 +82,12 @@ module Karafka
   end
 end
 
-Zeitwerk::Loader
-  .for_gem
-  .tap(&:setup)
-  .tap(&:eager_load)
-
-Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
+loader = Zeitwerk::Loader.for_gem
+# Do not load Rails extensions by default, this will be handled by Railtie if they are needed
+loader.ignore(Karafka.gem_root.join('lib/active_job'))
+# Do not load pro components, this will be handled by license manager
+loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
+loader.setup
+loader.eager_load