karafka 2.0.0.alpha5 → 2.0.0.beta2

data/lib/karafka/processing/jobs_queue.rb

@@ -12,7 +12,7 @@ module Karafka
     class JobsQueue
       # @return [Karafka::Processing::JobsQueue]
       def initialize
-        @queue = ::Queue.new
+        @queue = Queue.new
         # Those queues will act as a semaphores internally. Since we need an indicator for waiting
         # we could use Thread.pass but this is expensive. Instead we can just lock until any
         # of the workers finishes their work and we can re-check. This means that in the worse
@@ -21,7 +21,7 @@ module Karafka
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
         @semaphores = Hash.new { |h, k| h[k] = Queue.new }
-        @in_processing = Hash.new { |h, k| h[k] = {} }
+        @in_processing = Hash.new { |h, k| h[k] = [] }
         @mutex = Mutex.new
       end
 
@@ -44,9 +44,9 @@ module Karafka
         @mutex.synchronize do
           group = @in_processing[job.group_id]
 
-          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.key?(job.id)
+          raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
 
-          group[job.id] = true
+          group << job
         end
 
         @queue << job
@@ -60,14 +60,21 @@ module Karafka
         @queue.pop
       end
 
+      # Causes the wait lock to re-check the lock conditions and potential unlock.
+      # @param group_id [String] id of the group we want to unlock for one tick
+      # @note This does not release the wait lock. It just causes a conditions recheck
+      def tick(group_id)
+        @semaphores[group_id] << true
+      end
+
       # Marks a given job from a given group as completed. When there are no more jobs from a given
       # group to be executed, we won't wait.
       #
       # @param [Jobs::Base] job that was completed
       def complete(job)
         @mutex.synchronize do
-          @in_processing[job.group_id].delete(job.id)
-          @semaphores[job.group_id] << true
+          @in_processing[job.group_id].delete(job)
+          tick(job.group_id)
         end
       end
 
@@ -79,7 +86,7 @@ module Karafka
         @mutex.synchronize do
           @in_processing[group_id].clear
           # We unlock it just in case it was blocked when clearing started
-          @semaphores[group_id] << true
+          tick(group_id)
         end
       end
 
@@ -93,8 +100,17 @@ module Karafka
         end
       end
 
-      # Blocks when there are things in the queue in a given group and waits until all the jobs
-      #   from a given group are completed
+      # @param group_id [String]
+      #
+      # @return [Boolean] tell us if we have anything in the processing (or for processing) from
+      # a given group.
+      def empty?(group_id)
+        @in_processing[group_id].empty?
+      end
+
+      # Blocks when there are things in the queue in a given group and waits until all the blocking
+      #   jobs from a given group are completed
+      #
       # @param group_id [String] id of the group in which jobs we're interested.
       # @note This method is blocking.
       def wait(group_id)
@@ -107,14 +123,10 @@ module Karafka
 
       # @param group_id [String] id of the group in which jobs we're interested.
       # @return [Boolean] should we keep waiting or not
+      # @note We do not wait for non-blocking jobs. Their flow should allow for `poll` running
+      #   as they may exceed `max.poll.interval`
       def wait?(group_id)
-        # If it is stopping, all the previous messages that are processed at the moment need to
-        # finish. Otherwise we may risk closing the client and committing offsets afterwards
-        return false if Karafka::App.stopping? && @in_processing[group_id].empty?
-        return false if @queue.closed?
-        return false if @in_processing[group_id].empty?
-
-        true
+        !@in_processing[group_id].all?(&:non_blocking?)
       end
     end
   end

data/lib/karafka/processing/worker.rb

@@ -4,25 +4,35 @@ module Karafka
   module Processing
     # Workers are used to run jobs in separate threads.
     # Workers are the main processing units of the Karafka framework.
+    #
+    # Each job runs in three stages:
+    #   - prepare - here we can run any code that we would need to run blocking before we allow
+    #               the job to run fully async (non blocking). This will always run in a blocking
+    #               way and can be used to make sure all the resources and external dependencies
+    #               are satisfied before going async.
+    #
+    #   - call - actual processing logic that can run sync or async
+    #
+    #   - teardown - it should include any code that we want to run after we executed the user
+    #                code. This can be used to unlock certain resources or do other things that are
+    #                not user code but need to run after user code base is executed.
     class Worker
-      extend Forwardable
-
-      def_delegators :@thread, :join, :terminate, :alive?
+      include Helpers::Async
 
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
         @jobs_queue = jobs_queue
-        @thread = Thread.new do
-          # If anything goes wrong in this worker thread, it means something went really wrong and
-          # we should terminate.
-          Thread.current.abort_on_exception = true
-          loop { break unless process }
-        end
       end
 
       private
 
+      # Runs processing of jobs in a loop
+      # Stops when queue is closed.
+      def call
+        loop { break unless process }
+      end
+
       # Fetches a single job, processes it and marks as completed.
       #
       # @note We do not have error handling here, as no errors should propagate this far. If they
@@ -33,7 +43,18 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
+          job.prepare
+
+          # If a job is marked as non blocking, we can run a tick in the job queue and if there
+          # are no other blocking factors, the job queue will be unlocked.
+          # If this does not run, all the things will be blocking and job queue won't allow to
+          # pass it until done.
+          @jobs_queue.tick(job.group_id) if job.non_blocking?
+
           job.call
+
+          job.teardown
+
           true
         else
           false

data/lib/karafka/processing/workers_batch.rb

@@ -17,6 +17,11 @@ module Karafka
       def each(&block)
         @batch.each(&block)
       end
+
+      # @return [Integer] number of workers in the batch
+      def size
+        @batch.size
+      end
     end
   end
 end

data/lib/karafka/railtie.rb

@@ -82,8 +82,20 @@ if rails
       initializer 'karafka.require_karafka_boot_file' do |app|
         rails6plus = Rails.gem_version >= Gem::Version.new('6.0.0')
 
+        # If the boot file location is set to "false", we should not raise an exception and we
+        # should just not load karafka stuff. Setting this explicitly to false indicates, that
+        # karafka is part of the supply chain but it is not a first class citizen of a given
+        # system (may be just a dependency of a dependency), thus railtie should not kick in to
+        # load the non-existing boot file
+        next if Karafka.boot_file.to_s == 'false'
+
         karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
 
+        # Provide more comprehensive error for when no boot file
+        unless File.exist?(karafka_boot_file)
+          raise(Karafka::Errors::MissingBootFileError, karafka_boot_file)
+        end
+
         if rails6plus
           app.reloader.to_prepare do
             # Load Karafka boot file, so it can be used in Rails server context

data/lib/karafka/routing/consumer_group.rb

@@ -17,7 +17,7 @@ module Karafka
       def initialize(name)
         @name = name
         @id = Karafka::App.config.consumer_mapper.call(name)
-        @topics = []
+        @topics = Topics.new([])
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process

data/lib/karafka/routing/subscription_group.rb

@@ -10,7 +10,7 @@ module Karafka
     class SubscriptionGroup
       attr_reader :id, :topics
 
-      # @param topics [Array<Topic>] all the topics that share the same key settings
+      # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(topics)
         @id = SecureRandom.uuid

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -23,8 +23,8 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
-      # @param topics [Array<Topic>] array with topics based on which we want to build subscription
-      #   groups
+      # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
+      #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
       def call(topics)
         topics
@@ -32,6 +32,7 @@ module Karafka
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
+          .map { |topics_array| Routing::Topics.new(topics_array) }
           .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
       end
 

data/lib/karafka/routing/topics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Abstraction layer on top of groups of topics
+    class Topics
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+
+      # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
+      def initialize(topics_array)
+        @accumulator = topics_array.dup
+      end
+
+      # Yields each topic
+      #
+      # @param [Proc] block we want to yield with on each topic
+      def each(&block)
+        @accumulator.each(&block)
+      end
+
+      # Finds topic by its name
+      #
+      # @param topic_name [String] topic name
+      # @return [Karafka::Routing::Topic]
+      # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+      #   please create an issue.
+      def find(topic_name)
+        @accumulator.find { |topic| topic.name == topic_name } ||
+          raise(Karafka::Errors::TopicNotFoundError, topic_name)
+      end
+    end
+  end
+end

data/lib/karafka/runner.rb

@@ -3,32 +3,37 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end. If they do, it is a critical error
+    # Starts listening on all the listeners asynchronously and handles the jobs queue closing
+    # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
       jobs_queue = Processing::JobsQueue.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
-      Karafka::Server.workers = workers
+      listeners = Connection::ListenersBatch.new(jobs_queue)
 
-      threads = listeners(jobs_queue).map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
+      workers.each(&:async_call)
+      listeners.each(&:async_call)
 
       # We aggregate threads here for a supervised shutdown process
-      Karafka::Server.consumer_threads = threads
+      Karafka::Server.workers = workers
+      Karafka::Server.listeners = listeners
 
       # All the listener threads need to finish
-      threads.each(&:join)
+      listeners.each(&:join)
+
+      # We close the jobs queue only when no listener threads are working.
+      # This ensures, that everything was closed prior to us not accepting anymore jobs and that
+      # no more jobs will be enqueued. Since each listener waits for jobs to finish, once those
+      # are done, we can close.
+      jobs_queue.close
+
       # All the workers need to stop processing anything before we can stop the runner completely
+      # This ensures that even async long-running jobs have time to finish before we are done
+      # with everything. One thing worth keeping in mind though: It is the end user responsibility
+      # to handle the shutdown detection in their long-running processes. Otherwise if timeout
+      # is exceeded, there will be a forced shutdown.
       workers.each(&:join)
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something terrible happened
@@ -42,18 +47,5 @@ module Karafka
       Karafka::App.stop!
       raise e
     end
-
-    private
-
-    # @param jobs_queue [Processing::JobsQueue] the main processing queue
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages for each
-    #   of the subscription groups
-    def listeners(jobs_queue)
-      App
-        .subscription_groups
-        .map do |subscription_group|
-          Karafka::Connection::Listener.new(subscription_group, jobs_queue)
-        end
-    end
   end
 end

data/lib/karafka/scheduler.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  # FIFO scheduler for messages coming from various topics and partitions
+  class Scheduler
+    # Schedules jobs in the fifo order
+    #
+    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+    def schedule_consumption(queue, jobs_array)
+      jobs_array.each do |job|
+        queue << job
+      end
+    end
+
+    # Both revocation and shutdown jobs can also run in fifo by default
+    alias schedule_revocation schedule_consumption
+    alias schedule_shutdown schedule_consumption
+  end
+end

data/lib/karafka/server.rb

@@ -15,7 +15,7 @@ module Karafka
 
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
-      attr_accessor :consumer_threads
+      attr_accessor :listeners
 
       # Set of workers
       attr_accessor :workers
@@ -25,9 +25,12 @@ module Karafka
 
       # Method which runs app
       def run
-        process.on_sigint { stop }
-        process.on_sigquit { stop }
-        process.on_sigterm { stop }
+        # Since we do a lot of threading and queuing, we don't want to stop from the trap context
+        # as some things may not work there as expected, that is why we spawn a separate thread to
+        # handle the stopping process
+        process.on_sigint { Thread.new { stop } }
+        process.on_sigquit { Thread.new { stop } }
+        process.on_sigterm { Thread.new { stop } }
 
         # Start is blocking until stop is called and when we stop, it will wait until
         # all of the things are ready to stop
@@ -35,6 +38,8 @@ module Karafka
 
         # We always need to wait for Karafka to stop here since we should wait for the stop running
         # in a separate thread (or trap context) to indicate everything is closed
+        # Since `#start` is blocking, we were get here only after the runner is done. This will
+        # not add any performance degradation because of that.
         Thread.pass until Karafka::App.stopped?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
@@ -70,16 +75,16 @@ module Karafka
       def stop
         Karafka::App.stop!
 
-        timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
+        timeout = Karafka::App.config.shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if consumer_threads.count(&:alive?).zero? &&
+          if listeners.count(&:alive?).zero? &&
              workers.count(&:alive?).zero?
 
-            Thread.new { Karafka::App.producer.close }.join
+            Karafka::App.producer.close
 
             return
           end
@@ -89,22 +94,18 @@ module Karafka
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
-        thread = Thread.new do
-          Karafka.monitor.instrument(
-            'error.occurred',
-            caller: self,
-            error: e,
-            type: 'app.stopping.error'
-          )
-
-          # We're done waiting, lets kill them!
-          workers.each(&:terminate)
-          consumer_threads.each(&:terminate)
-
-          Karafka::App.producer.close
-        end
-
-        thread.join
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'app.stopping.error'
+        )
+
+        # We're done waiting, lets kill them!
+        workers.each(&:terminate)
+        listeners.each(&:terminate)
+
+        Karafka::App.producer.close
 
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit! FORCEFUL_EXIT_CODE

data/lib/karafka/setup/config.rb

@@ -60,7 +60,7 @@ module Karafka
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
-      setting :max_messages, default: 100_000
+      setting :max_messages, default: 1_000
       # option [Integer] number of milliseconds we can wait while fetching data
       setting :max_wait_time, default: 10_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no
@@ -96,6 +96,8 @@ module Karafka
         # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
         #   group builder
         setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+        # option scheduler [Class] scheduler we will be using
+        setting :scheduler, default: Scheduler.new
 
         # Karafka components for ActiveJob
         setting :active_job do
@@ -115,6 +117,7 @@ module Karafka
         def setup(&block)
           configure(&block)
           merge_kafka_defaults!(config)
+
           Contracts::Config.new.validate!(config.to_h)
 
           # Check the license presence (if needed) and

data/lib/karafka/time_trackers/pause.rb

@@ -41,9 +41,12 @@ module Karafka
 
       # Pauses the processing from now till the end of the interval (backoff or non-backoff)
       # and records the count.
-      def pause
+      # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
+      # @note Providing this value can be useful when we explicitly want to pause for a certain
+      #   period of time, outside of any regular pausing logic
+      def pause(timeout = backoff_interval)
         @started_at = now
-        @ends_at = @started_at + backoff_interval
+        @ends_at = @started_at + timeout
         @count += 1
       end
 
@@ -53,6 +56,11 @@ module Karafka
         @ends_at = nil
       end
 
+      # Expires the pause, so it can be considered expired
+      def expire
+        @ends_at = nil
+      end
+
       # @return [Boolean] are we paused from processing
       def paused?
         !@started_at.nil?

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.alpha5'
+  VERSION = '2.0.0.beta2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.alpha5
+  version: 2.0.0.beta2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-04-03 00:00:00.000000000 Z
+date: 2022-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -184,7 +184,7 @@ files:
 - lib/karafka/active_job/dispatcher.rb
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
-- lib/karafka/active_job/routing_extensions.rb
+- lib/karafka/active_job/routing/extensions.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -195,8 +195,10 @@ files:
 - lib/karafka/cli/server.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/listener.rb
+- lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
 - lib/karafka/connection/pauses_manager.rb
+- lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
@@ -206,6 +208,7 @@ files:
 - lib/karafka/contracts/server_cli_options.rb
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
+- lib/karafka/helpers/async.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation.rb
 - lib/karafka/instrumentation/callbacks/error.rb
@@ -225,9 +228,13 @@ files:
 - lib/karafka/messages/seek.rb
 - lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/pro.rb
+- lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
 - lib/karafka/pro/loader.rb
+- lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/scheduler.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
@@ -247,7 +254,9 @@ files:
 - lib/karafka/routing/subscription_group.rb
 - lib/karafka/routing/subscription_groups_builder.rb
 - lib/karafka/routing/topic.rb
+- lib/karafka/routing/topics.rb
 - lib/karafka/runner.rb
+- lib/karafka/scheduler.rb
 - lib/karafka/serialization/json/deserializer.rb
 - lib/karafka/server.rb
 - lib/karafka/setup/config.rb
@@ -282,7 +291,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.3.4
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Ruby based framework for working with Apache Kafka

data/lib/karafka/active_job/routing_extensions.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # ActiveJob related Karafka stuff
-  module ActiveJob
-    # Routing extensions for ActiveJob
-    module RoutingExtensions
-      # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
-      # the consumer class
-      # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-      def active_job_topic(name)
-        topic(name) do
-          consumer App.config.internal.active_job.consumer
-        end
-      end
-    end
-  end
-end