karafka 2.0.0.beta1 → 2.0.0.beta4

data/lib/karafka/runner.rb

@@ -3,32 +3,37 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end. If they do, it is a critical error
+    # Starts listening on all the listeners asynchronously and handles the jobs queue closing
+    # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
       jobs_queue = Processing::JobsQueue.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
-      Karafka::Server.workers = workers
+      listeners = Connection::ListenersBatch.new(jobs_queue)
 
-      threads = listeners(jobs_queue).map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
+      workers.each(&:async_call)
+      listeners.each(&:async_call)
 
       # We aggregate threads here for a supervised shutdown process
-      Karafka::Server.consumer_threads = threads
+      Karafka::Server.workers = workers
+      Karafka::Server.listeners = listeners
 
       # All the listener threads need to finish
-      threads.each(&:join)
+      listeners.each(&:join)
+
+      # We close the jobs queue only when no listener threads are working.
+      # This ensures, that everything was closed prior to us not accepting anymore jobs and that
+      # no more jobs will be enqueued. Since each listener waits for jobs to finish, once those
+      # are done, we can close.
+      jobs_queue.close
+
       # All the workers need to stop processing anything before we can stop the runner completely
+      # This ensures that even async long-running jobs have time to finish before we are done
+      # with everything. One thing worth keeping in mind though: It is the end user responsibility
+      # to handle the shutdown detection in their long-running processes. Otherwise if timeout
+      # is exceeded, there will be a forced shutdown.
       workers.each(&:join)
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something terrible happened
@@ -42,18 +47,5 @@ module Karafka
       Karafka::App.stop!
       raise e
     end
-
-    private
-
-    # @param jobs_queue [Processing::JobsQueue] the main processing queue
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages for each
-    #   of the subscription groups
-    def listeners(jobs_queue)
-      App
-        .subscription_groups
-        .map do |subscription_group|
-          Karafka::Connection::Listener.new(subscription_group, jobs_queue)
-        end
-    end
   end
 end

data/lib/karafka/scheduler.rb

@@ -3,19 +3,18 @@
 module Karafka
   # FIFO scheduler for messages coming from various topics and partitions
   class Scheduler
-    # Yields messages from partitions in the fifo order
+    # Schedules jobs in the fifo order
     #
-    # @param messages_buffer [Karafka::Connection::MessagesBuffer] messages buffer with data from
-    #   multiple topics and partitions
-    # @yieldparam [String] topic name
-    # @yieldparam [Integer] partition number
-    # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
-    def call(messages_buffer)
-      messages_buffer.each do |topic, partitions|
-        partitions.each do |partition, messages|
-          yield(topic, partition, messages)
-        end
+    # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+    # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+    def schedule_consumption(queue, jobs_array)
+      jobs_array.each do |job|
+        queue << job
       end
     end
+
+    # Both revocation and shutdown jobs can also run in fifo by default
+    alias schedule_revocation schedule_consumption
+    alias schedule_shutdown schedule_consumption
   end
 end

data/lib/karafka/server.rb

@@ -15,7 +15,7 @@ module Karafka
 
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
-      attr_accessor :consumer_threads
+      attr_accessor :listeners
 
       # Set of workers
       attr_accessor :workers
@@ -25,9 +25,12 @@ module Karafka
 
       # Method which runs app
       def run
-        process.on_sigint { stop }
-        process.on_sigquit { stop }
-        process.on_sigterm { stop }
+        # Since we do a lot of threading and queuing, we don't want to stop from the trap context
+        # as some things may not work there as expected, that is why we spawn a separate thread to
+        # handle the stopping process
+        process.on_sigint { Thread.new { stop } }
+        process.on_sigquit { Thread.new { stop } }
+        process.on_sigterm { Thread.new { stop } }
 
         # Start is blocking until stop is called and when we stop, it will wait until
         # all of the things are ready to stop
@@ -35,6 +38,8 @@ module Karafka
 
         # We always need to wait for Karafka to stop here since we should wait for the stop running
         # in a separate thread (or trap context) to indicate everything is closed
+        # Since `#start` is blocking, we were get here only after the runner is done. This will
+        # not add any performance degradation because of that.
         Thread.pass until Karafka::App.stopped?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
@@ -70,16 +75,16 @@ module Karafka
       def stop
         Karafka::App.stop!
 
-        timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
+        timeout = Karafka::App.config.shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if consumer_threads.count(&:alive?).zero? &&
+          if listeners.count(&:alive?).zero? &&
              workers.count(&:alive?).zero?
 
-            Thread.new { Karafka::App.producer.close }.join
+            Karafka::App.producer.close
 
             return
           end
@@ -89,22 +94,18 @@ module Karafka
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
-        thread = Thread.new do
-          Karafka.monitor.instrument(
-            'error.occurred',
-            caller: self,
-            error: e,
-            type: 'app.stopping.error'
-          )
-
-          # We're done waiting, lets kill them!
-          workers.each(&:terminate)
-          consumer_threads.each(&:terminate)
-
-          Karafka::App.producer.close
-        end
-
-        thread.join
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'app.stopping.error'
+        )
+
+        # We're done waiting, lets kill them!
+        workers.each(&:terminate)
+        listeners.each(&:terminate)
+
+        Karafka::App.producer.close
 
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit! FORCEFUL_EXIT_CODE

data/lib/karafka/setup/config.rb

@@ -83,7 +83,7 @@ module Karafka
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       setting :kafka, default: {}
 
-      # Namespace for internal settings that should not be modified
+      # Namespace for internal settings that should not be modified directly
       setting :internal do
         # option routing_builder [Karafka::Routing::Builder] builder instance
         setting :routing_builder, default: Routing::Builder.new
@@ -98,6 +98,8 @@ module Karafka
         setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
         # option scheduler [Class] scheduler we will be using
         setting :scheduler, default: Scheduler.new
+        # option jobs_builder [Class] jobs builder we want to use
+        setting :jobs_builder, default: Processing::JobsBuilder.new
 
         # Karafka components for ActiveJob
         setting :active_job do
@@ -117,6 +119,7 @@ module Karafka
         def setup(&block)
           configure(&block)
           merge_kafka_defaults!(config)
+
           Contracts::Config.new.validate!(config.to_h)
 
           # Check the license presence (if needed) and

data/lib/karafka/status.rb

@@ -31,9 +31,7 @@ module Karafka
         # We skip as during this state we do not have yet a monitor
         return if initializing?
 
-        # Trap context disallows to run certain things that we instrument
-        # so the state changes are executed from a separate thread
-        Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
+        Karafka.monitor.instrument("app.#{state}")
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta1'
+  VERSION = '2.0.0.beta4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta1
+  version: 2.0.0.beta4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-05-22 00:00:00.000000000 Z
+date: 2022-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -112,7 +112,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.3.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -171,7 +171,9 @@ files:
 - bin/create_token
 - bin/integrations
 - bin/karafka
+- bin/scenario
 - bin/stress
+- bin/wait_for_kafka
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
 - config/errors.yml
@@ -184,7 +186,7 @@ files:
 - lib/karafka/active_job/dispatcher.rb
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
-- lib/karafka/active_job/routing_extensions.rb
+- lib/karafka/active_job/routing/extensions.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -195,8 +197,10 @@ files:
 - lib/karafka/cli/server.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/listener.rb
+- lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
 - lib/karafka/connection/pauses_manager.rb
+- lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
@@ -206,6 +210,7 @@ files:
 - lib/karafka/contracts/server_cli_options.rb
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
+- lib/karafka/helpers/async.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation.rb
 - lib/karafka/instrumentation/callbacks/error.rb
@@ -225,10 +230,16 @@ files:
 - lib/karafka/messages/seek.rb
 - lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/pro.rb
+- lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
+- lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/routing/extensions.rb
+- lib/karafka/pro/scheduler.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
@@ -236,7 +247,9 @@ files:
 - lib/karafka/processing/jobs/consume.rb
 - lib/karafka/processing/jobs/revoked.rb
 - lib/karafka/processing/jobs/shutdown.rb
+- lib/karafka/processing/jobs_builder.rb
 - lib/karafka/processing/jobs_queue.rb
+- lib/karafka/processing/result.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
@@ -248,6 +261,7 @@ files:
 - lib/karafka/routing/subscription_group.rb
 - lib/karafka/routing/subscription_groups_builder.rb
 - lib/karafka/routing/topic.rb
+- lib/karafka/routing/topics.rb
 - lib/karafka/runner.rb
 - lib/karafka/scheduler.rb
 - lib/karafka/serialization/json/deserializer.rb
@@ -266,6 +280,7 @@ files:
 homepage: https://karafka.io
 licenses:
 - LGPL-3.0
+- Commercial
 metadata:
   source_code_uri: https://github.com/karafka/karafka
   rubygems_mfa_required: 'true'

data/lib/karafka/active_job/routing_extensions.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # ActiveJob related Karafka stuff
-  module ActiveJob
-    # Routing extensions for ActiveJob
-    module RoutingExtensions
-      # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
-      # the consumer class
-      # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-      def active_job_topic(name)
-        topic(name) do
-          consumer App.config.internal.active_job.consumer
-        end
-      end
-    end
-  end
-end