karafka 2.0.0.beta2 → 2.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f75623e7d9cdcc4ba151ad551079275528c56bf66cd9c32ecc585756a8d505c
-  data.tar.gz: e0becf53133b579f581ddfdf947bbff21221fe69a8c73a0406174aecd0155f3a
+  metadata.gz: 86b352cc7737bde8484567662d546165793496adb6a923e33aa7e4823e4a456f
+  data.tar.gz: 2a180ba7b177db9f4c67de387262d47ac3eb2580860a548992912cd89a049da7
 SHA512:
-  metadata.gz: 2ef2ac59f1ea60136abbaccf460206a0c2f6d4fe3124eda520f3a19568702acc774e0d9e02eae24cfe6bb6cb8ee8aa74602588aa818dc3537cd6bbc8409f159d
-  data.tar.gz: e29e964e777e2bd8a551458f591b92aea69a3ac2eafa1bc1d75bb42d7cd6bb904abf997724be4268ae1f9d429627d7b92d5c38e51817d36b0f27c6499a062af3
+  metadata.gz: 207700c3e1fab4d3370de7eddae72453fc180c65d817f4d7c021929327e5464fcd8a9bb1610c641080f82a60bc038e4517dc7bc86c0aa4c83852dd9467441e75
+  data.tar.gz: ab94239255ff841e0728c6e6585d4145a9c4f42dfd02d8df062975bd28ad3caf2cef50f5488d30a1356ad49e2529f772379bc947ba5804a15f12e56989cf9a7a

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Karafka framework changelog
 
+## 2.0.0-beta3 (2022-06-14)
+- Jobs building responsibility extracted out of the listener code base.
+- Fix a case where specs supervisor would try to kill no longer running process (#868)
+- Fix an instable integration spec that could misbehave under load
+- Commit offsets prior to pausing partitions to ensure that the latest offset is always committed
+- Fix a case where consecutive CTRL+C (non-stop) would case an exception during forced shutdown
+- Add missing `consumer.prepared.error` into `LoggerListener`
+- Delegate partition resuming from the consumers to listeners threads.
+- Add support for Long Running Jobs (LRJ) for ActiveJob [PRO]
+- Add support for Long Running Jobs for consumers [PRO]
+- Allow `active_job_topic` to accept a block for extra topic related settings
+- Remove no longer needed logger threads
+- Auto-adapt number of processes for integration specs based on the number of CPUs
+- Introduce an integration spec runner that prints everything to stdout (better for development)
+- Introduce extra integration specs for various ActiveJob usage scenarios
+- Rename consumer method `#prepared` to `#prepare` to reflect better its use-case
+- For test and dev raise an error when expired license key is used (never for non dev)
+- Add worker related monitor events (`worker.process` and `worker.processed`)
+- Update `LoggerListener` to include more useful information about processing and polling messages
+
 ## 2.0.0-beta2 (2022-06-07)
 - Abstract away notion of topics groups (until now it was just an array)
 - Optimize how jobs queue is closed. Since we enqueue jobs only from the listeners, we can safely close jobs queue once listeners are done. By extracting this responsibility from listeners, we remove corner cases and race conditions. Note here: for non-blocking jobs we do wait for them to finish while running the `poll`. This ensures, that for async jobs that are long-living, we do not reach `max.poll.interval`.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.beta2)
+    karafka (2.0.0.beta3)
       dry-configurable (~> 0.13)
       dry-monitor (~> 0.5)
       dry-validation (~> 1.7)
@@ -45,7 +45,7 @@ GEM
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
       dry-events (~> 0.2)
-    dry-schema (1.9.1)
+    dry-schema (1.9.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.13, >= 0.13.0)
       dry-core (~> 0.5, >= 0.5)
@@ -58,12 +58,12 @@ GEM
       dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.8.0)
+    dry-validation (1.8.1)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.7, >= 0.7.1)
       dry-core (~> 0.5, >= 0.5)
       dry-initializer (~> 3.0)
-      dry-schema (~> 1.9, >= 1.9.1)
+      dry-schema (~> 1.8, >= 1.8.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)

data/bin/integrations

@@ -11,21 +11,21 @@ require 'open3'
 require 'fileutils'
 require 'pathname'
 require 'tmpdir'
+require 'etc'
 
 ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
 
-# Raised from the parent process if any of the integration tests fails
-IntegrationTestError = Class.new(StandardError)
-
 # How many child processes with integration specs do we want to run in parallel
-# When the value is high, there's a problem with thread allocation on Github
-CONCURRENCY = 4
+# When the value is high, there's a problem with thread allocation on Github CI, tht is why
+# we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
+# of CPU
+CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 2
 
 # Abstraction around a single test scenario execution process
 class Scenario
   # How long a scenario can run before we kill it
   # This is a fail-safe just in case something would hang
-  MAX_RUN_TIME = 60 * 2
+  MAX_RUN_TIME = 3 * 60 # 3 minutes tops
 
   # There are rare cases where Karafka may force shutdown for some of the integration cases
   # This includes exactly those
@@ -73,7 +73,12 @@ class Scenario
     # If the thread is running too long, kill it
     if current_time - @started_at > MAX_RUN_TIME
       @wait_thr.kill
-      Process.kill('TERM', pid)
+
+      begin
+        Process.kill('TERM', pid)
+      # It may finish right after we want to kill it, that's why we ignore this
+      rescue Errno::ESRCH
+      end
     end
 
     # We read it so it won't grow as we use our default logger that prints to both test.log and
@@ -106,14 +111,15 @@ class Scenario
 
   # Prints a status report when scenario is finished and stdout if it failed
   def report
-    result = success? ? "\e[#{32}m#{'OK'}\e[0m" : "\e[#{31}m#{'FAILED'}\e[0m"
-
-    puts "#{result} #{name}"
-
-    unless success?
+    if success?
+      print "\e[#{32}m#{'.'}\e[0m"
+    else
+      puts
+      puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
       puts "Exit code: #{exit_code}"
       puts @stdout_tail
       puts @stderr.read
+      puts
     end
   end
 
@@ -204,5 +210,21 @@ while finished_scenarios.size < scenarios.size
   sleep(0.1)
 end
 
-# Fail all if any of the tests does not have expected exit code
-raise IntegrationTestError unless finished_scenarios.all?(&:success?)
+failed_scenarios = finished_scenarios.reject(&:success?)
+
+# Report once more on the failed jobs
+# This will only list scenarios that failed without printing their stdout here.
+if failed_scenarios.empty?
+  puts
+else
+  puts "\nFailed scenarios:\n\n"
+
+  failed_scenarios.each do |scenario|
+    puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
+  end
+
+  puts
+
+  # Exit with 1 if not all scenarios were successful
+  exit 1
+end

data/bin/scenario

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+# Runner for non-parallel execution of a single scenario.
+# It prints all the info stdout, etc and basically replaces itself with the scenario execution.
+# It is useful when we work with a single spec and we need all the debug info
+
+raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
+
+require 'open3'
+require 'fileutils'
+require 'pathname'
+require 'tmpdir'
+require 'etc'
+
+ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
+
+# Load all the specs
+specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
+
+# If filters is provided, apply
+# Allows to provide several filters one after another and applies all of them
+ARGV.each do |filter|
+  specs.delete_if { |name| !name.include?(filter) }
+end
+
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
+raise ArgumentError, "Many specs found with filters: #{ARGV.join(', ')}" if specs.size != 1
+
+exec("bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{specs[0]}")

data/docker-compose.yml

@@ -17,6 +17,8 @@ services:
         "integrations_0_02:2:1,\
          integrations_1_02:2:1,\
          integrations_2_02:2:1,\
+         integrations_3_02:2:1,\
+         integrations_4_02:2:1,\
          integrations_0_03:3:1,\
          integrations_1_03:3:1,\
          integrations_2_03:3:1,\

data/karafka.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.homepage    = 'https://karafka.io'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
-  spec.license     = 'LGPL-3.0'
+  spec.licenses    = ['LGPL-3.0', 'Commercial']
 
   spec.add_dependency 'dry-configurable', '~> 0.13'
   spec.add_dependency 'dry-monitor', '~> 0.5'

data/lib/karafka/active_job/routing/extensions.rb

@@ -10,9 +10,19 @@ module Karafka
         # This method simplifies routes definition for ActiveJob topics / queues by auto-injecting
         # the consumer class
         # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
-        def active_job_topic(name)
+        # @param block [Proc] block that we can use for some extra configuration
+        def active_job_topic(name, &block)
           topic(name) do
             consumer App.config.internal.active_job.consumer
+
+            next unless block
+
+            instance_eval(&block)
+
+            # This is handled by our custom ActiveJob consumer
+            # Without this, default behaviour would cause messages to skip upon shutdown as the
+            # offset would be committed for the last message
+            manual_offset_management true
           end
         end
       end

data/lib/karafka/base_consumer.rb

@@ -15,8 +15,33 @@ module Karafka
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
 
+    # Can be used to run preparation code
+    #
+    # @private
+    # @note This should not be used by the end users as it is part of the lifecycle of things but
+    #   not as part of the public api. This can act as a hook when creating non-blocking
+    #   consumers and doing other advanced stuff
+    def on_prepare
+      Karafka.monitor.instrument('consumer.prepared', caller: self) do
+        prepare
+      end
+
+      true
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.prepare.error'
+      )
+
+      false
+    end
+
     # Executes the default consumer flow.
     #
+    # @return [Boolean] true if there was no exception, otherwise false.
+    #
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
@@ -33,6 +58,8 @@ module Karafka
         # with manual offset management
         mark_as_consumed(messages.last)
       end
+
+      true
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -42,6 +69,8 @@ module Karafka
       )
 
       pause(@seek_offset || messages.first.offset)
+
+      false
     end
 
     # Trigger method for running on shutdown.
@@ -51,6 +80,8 @@ module Karafka
       Karafka.monitor.instrument('consumer.revoked', caller: self) do
         revoked
       end
+
+      true
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -58,6 +89,8 @@ module Karafka
         caller: self,
         type: 'consumer.revoked.error'
       )
+
+      false
     end
 
     # Trigger method for running on shutdown.
@@ -67,6 +100,8 @@ module Karafka
       Karafka.monitor.instrument('consumer.shutdown', caller: self) do
         shutdown
       end
+
+      true
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -74,32 +109,15 @@ module Karafka
         caller: self,
         type: 'consumer.shutdown.error'
       )
-    end
 
-    # Can be used to run preparation code
-    #
-    # @private
-    # @note This should not be used by the end users as it is part of the lifecycle of things but
-    #   not as part of the public api. This can act as a hook when creating non-blocking
-    #   consumers and doing other advanced stuff
-    def on_prepared
-      Karafka.monitor.instrument('consumer.prepared', caller: self) do
-        prepared
-      end
-    rescue StandardError => e
-      Karafka.monitor.instrument(
-        'error.occurred',
-        error: e,
-        caller: self,
-        type: 'consumer.prepared.error'
-      )
+      false
     end
 
     private
 
     # Method that gets called in the blocking flow allowing to setup any type of resources or to
     # send additional commands to Kafka before the proper execution starts.
-    def prepared; end
+    def prepare; end
 
     # Method that will perform business logic and on data received from Kafka (it will consume
     #   the data)
@@ -144,22 +162,19 @@ module Karafka
     # @param timeout [Integer, nil] how long in milliseconds do we want to pause or nil to use the
     #   default exponential pausing strategy defined for retries
     def pause(offset, timeout = nil)
+      timeout ? pause_tracker.pause(timeout) : pause_tracker.pause
+
       client.pause(
         messages.metadata.topic,
         messages.metadata.partition,
         offset
       )
-
-      timeout ? pause_tracker.pause(timeout) : pause_tracker.pause
     end
 
     # Resumes processing of the current topic partition
     def resume
-      client.resume(
-        messages.metadata.topic,
-        messages.metadata.partition
-      )
-
+      # This is sufficient to expire a partition pause, as with it will be resumed by the listener
+      # thread before the next poll.
       pause_tracker.expire
     end
 

data/lib/karafka/connection/client.rb

@@ -86,8 +86,7 @@ module Karafka
       # @param message [Karafka::Messages::Message]
       def store_offset(message)
         @mutex.synchronize do
-          @offsetting = true
-          @kafka.store_offset(message)
+          internal_store_offset(message)
         end
       end
 
@@ -104,14 +103,7 @@ module Karafka
       def commit_offsets(async: true)
         @mutex.lock
 
-        return unless @offsetting
-
-        @kafka.commit(nil, async)
-        @offsetting = false
-      rescue Rdkafka::RdkafkaError => e
-        return if e.code == :no_offset
-
-        raise e
+        internal_commit_offsets(async: async)
       ensure
         @mutex.unlock
       end
@@ -128,7 +120,11 @@ module Karafka
       #
       # @param message [Messages::Message, Messages::Seek] message to which we want to seek to
       def seek(message)
+        @mutex.lock
+
         @kafka.seek(message)
+      ensure
+        @mutex.unlock
       end
 
       # Pauses given partition and moves back to last successful offset processed.
@@ -144,15 +140,17 @@ module Karafka
         # Do not pause if the client got closed, would not change anything
         return if @closed
 
+        pause_msg = Messages::Seek.new(topic, partition, offset)
+
+        internal_commit_offsets(async: false)
+
         tpl = topic_partition_list(topic, partition)
 
         return unless tpl
 
         @kafka.pause(tpl)
 
-        pause_msg = Messages::Seek.new(topic, partition, offset)
-
-        seek(pause_msg)
+        @kafka.seek(pause_msg)
       ensure
         @mutex.unlock
       end
@@ -166,6 +164,11 @@ module Karafka
 
         return if @closed
 
+        # Always commit synchronously offsets if any when we resume
+        # This prevents resuming without offset in case it would not be committed prior
+        # We can skip performance penalty since resuming should not happen too often
+        internal_commit_offsets(async: false)
+
         tpl = topic_partition_list(topic, partition)
 
         return unless tpl
@@ -214,11 +217,31 @@ module Karafka
 
       private
 
+      # Non thread-safe offset storing method
+      # @param message [Karafka::Messages::Message]
+      def internal_store_offset(message)
+        @offsetting = true
+        @kafka.store_offset(message)
+      end
+
+      # Non thread-safe message committing method
+      # @param async [Boolean] should the commit happen async or sync (async by default)
+      def internal_commit_offsets(async: true)
+        return unless @offsetting
+
+        @kafka.commit(nil, async)
+        @offsetting = false
+      rescue Rdkafka::RdkafkaError => e
+        return if e.code == :no_offset
+
+        raise e
+      end
+
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
-        commit_offsets!
-
         @mutex.synchronize do
+          internal_commit_offsets(async: false)
+
           @closed = true
 
           # Remove callbacks runners that were registered

data/lib/karafka/connection/listener.rb

@@ -10,12 +10,18 @@ module Karafka
     class Listener
       include Helpers::Async
 
+      # Can be useful for logging
+      # @return [String] id of this listener
+      attr_reader :id
+
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
       def initialize(subscription_group, jobs_queue)
+        @id = SecureRandom.uuid
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
+        @jobs_builder = ::Karafka::App.config.internal.jobs_builder
         @pauses_manager = PausesManager.new
         @client = Client.new(@subscription_group)
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
@@ -62,16 +68,16 @@ module Karafka
 
           resume_paused_partitions
 
-          # We need to fetch data before we revoke lost partitions details as during the polling
-          # the callbacks for tracking lost partitions are triggered. Otherwise we would be always
-          # one batch behind.
-          poll_and_remap_messages
-
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop.received',
             caller: self,
             messages_buffer: @messages_buffer
-          )
+          ) do
+            # We need to fetch data before we revoke lost partitions details as during the polling
+            # the callbacks for tracking lost partitions are triggered. Otherwise we would be
+            # always one batch behind.
+            poll_and_remap_messages
+          end
 
           # If there were revoked partitions, we need to wait on their jobs to finish before
           # distributing consuming jobs as upon revoking, we might get assigned to the same
@@ -148,7 +154,7 @@ module Karafka
           partitions.each do |partition|
             pause_tracker = @pauses_manager.fetch(topic, partition)
             executor = @executors.fetch(topic, partition, pause_tracker)
-            jobs << Processing::Jobs::Revoked.new(executor)
+            jobs << @jobs_builder.revoked(executor)
           end
         end
 
@@ -160,7 +166,7 @@ module Karafka
         jobs = []
 
         @executors.each do |_, _, executor|
-          jobs << Processing::Jobs::Shutdown.new(executor)
+          jobs << @jobs_builder.shutdown(executor)
         end
 
         @scheduler.schedule_shutdown(@jobs_queue, jobs)
@@ -185,13 +191,11 @@ module Karafka
         jobs = []
 
         @messages_buffer.each do |topic, partition, messages|
-          pause = @pauses_manager.fetch(topic, partition)
-
-          next if pause.paused?
+          pause_tracker = @pauses_manager.fetch(topic, partition)
 
-          executor = @executors.fetch(topic, partition, pause)
+          executor = @executors.fetch(topic, partition, pause_tracker)
 
-          jobs << Processing::Jobs::Consume.new(executor, messages)
+          jobs << @jobs_builder.consume(executor, messages)
         end
 
         @scheduler.schedule_consumption(@jobs_queue, jobs)

data/lib/karafka/contracts/config.rb

@@ -31,10 +31,11 @@ module Karafka
         # We validate internals just to be sure, that they are present and working
         required(:internal).schema do
           required(:routing_builder)
+          required(:subscription_groups_builder)
+          required(:jobs_builder)
           required(:status)
           required(:process)
           required(:scheduler)
-          required(:subscription_groups_builder)
         end
       end
 

data/lib/karafka/instrumentation/logger_listener.rb

@@ -15,16 +15,43 @@ module Karafka
 
       # Logs each messages fetching attempt
       #
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_connection_listener_fetch_loop(_event)
-        info 'Receiving new messages from Kafka...'
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_listener_fetch_loop(event)
+        listener = event[:caller]
+        info "[#{listener.id}] Polling messages..."
       end
 
       # Logs about messages that we've received from Kafka
       #
       # @param event [Dry::Events::Event] event details including payload
       def on_connection_listener_fetch_loop_received(event)
-        info "Received #{event[:messages_buffer].size} new messages from Kafka"
+        listener = event[:caller]
+        time = event[:time]
+        messages_count = event[:messages_buffer].size
+        info "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
+      end
+
+      # Prints info about the fact that a given job has started
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_worker_process(event)
+        job = event[:job]
+        job_type = job.class.to_s.split('::').last
+        consumer = job.executor.topic.consumer
+        topic = job.executor.topic.name
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
+      end
+
+      # Prints info about the fact that a given job has finished
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_worker_processed(event)
+        job = event[:job]
+        time = event[:time]
+        job_type = job.class.to_s.split('::').last
+        consumer = job.executor.topic.consumer
+        topic = job.executor.topic.name
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
       # Logs info about system signals that Karafka received.
@@ -52,16 +79,14 @@ module Karafka
       #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_stopping(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { info 'Stopping Karafka server' }
+        info 'Stopping Karafka server'
       end
 
       # Logs info that we stopped the Karafka server.
       #
       # @param _event [Dry::Events::Event] event details including payload
       def on_app_stopped(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { info 'Stopped Karafka server' }
+        info 'Stopped Karafka server'
       end
 
       # There are many types of errors that can occur in many places, but we provide a single
@@ -73,6 +98,9 @@ module Karafka
         details = (error.backtrace || []).join("\n")
 
         case type
+        when 'consumer.prepared.error'
+          error "Consumer prepared error: #{error}"
+          error details
         when 'consumer.consume.error'
           error "Consumer consuming error: #{error}"
           error details
@@ -95,8 +123,7 @@ module Karafka
           fatal "Runner crashed due to an error: #{error}"
           fatal details
         when 'app.stopping.error'
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { error 'Forceful Karafka server stop' }
+          error 'Forceful Karafka server stop'
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details

data/lib/karafka/instrumentation/monitor.rb

@@ -33,6 +33,9 @@ module Karafka
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
+        worker.process
+        worker.processed
+
         statistics.emitted
 
         error.occurred