karafka 2.0.0.beta2 → 2.0.0.beta3

data/lib/karafka/licenser.rb

@@ -33,6 +33,8 @@ module Karafka
 
       return if license_config.expires_on > Date.today
 
+      raise_expired_license_token_in_dev(license_config.expires_on)
+
       notify_if_license_expired(license_config.expires_on)
     end
 
@@ -53,24 +55,41 @@ module Karafka
       )
     end
 
+    # Raises an error for test and dev environments if running pro with expired license
+    # We never want to cause any non-dev problems and we should never crash anything else than
+    # tests and development envs.
+    #
+    # @param expires_on [Date] when the license expires
+    def raise_expired_license_token_in_dev(expires_on)
+      env = Karafka::App.env
+
+      return unless env.development? || env.test?
+
+      raise Errors::ExpiredLicenseTokenError.new, expired_message(expires_on)
+    end
+
     # We do not raise an error here as we don't want to cause any problems to someone that runs
     # Karafka on production. Error message is enough.
     #
     # @param expires_on [Date] when the license expires
     def notify_if_license_expired(expires_on)
-      message = <<~MSG.tr("\n", ' ')
-        Your license expired on #{expires_on}.
-        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-      MSG
-
-      Karafka.logger.error(message)
+      Karafka.logger.error(expired_message(expires_on))
 
       Karafka.monitor.instrument(
         'error.occurred',
         caller: self,
-        error: Errors::ExpiredLicenseTokenError.new(message),
+        error: Errors::ExpiredLicenseTokenError.new(expired_message(expires_on)),
         type: 'licenser.expired'
       )
     end
+
+    # @param expires_on [Date] when the license expires
+    # @return [String] expired message
+    def expired_message(expires_on)
+      <<~MSG.tr("\n", ' ')
+        Your license expired on #{expires_on}.
+        Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+      MSG
+    end
   end
 end

data/lib/karafka/pro/active_job/consumer.rb

@@ -1,20 +1,47 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     module ActiveJob
-      # This Karafka component is a Pro component.
-      # All of the commercial components are present in the lib/karafka/pro directory of this
-      # repository and their usage requires commercial license agreement.
-      #
-      # Karafka has also commercial-friendly license, commercial support and commercial components.
-      #
-      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-      # of your code to Maciej Mensfeld.
-
       # Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
       # running jobs
+      #
+      # When in LRJ, it will pause a given partition forever and will resume its processing only
+      # when all the jobs are done processing.
+      #
+      # It contains slightly better revocation warranties than the regular blocking consumer as
+      # it can stop processing batch of jobs in the middle after the revocation.
       class Consumer < Karafka::ActiveJob::Consumer
+        # Runs ActiveJob jobs processing and handles lrj if needed
+        def consume
+          messages.each do |message|
+            # If for any reason we've lost this partition, not worth iterating over new messages
+            # as they are no longer ours
+            return if revoked?
+            break if Karafka::App.stopping?
+
+            ::ActiveJob::Base.execute(
+              ::ActiveSupport::JSON.decode(message.raw_payload)
+            )
+
+            # We check it twice as the job may be long running
+            return if revoked?
+
+            mark_as_consumed(message)
+
+            # Do not process more if we are shutting down
+            break if Karafka::App.stopping?
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     # Karafka Pro ActiveJob components
     module ActiveJob
-      # This Karafka component is a Pro component.
-      # All of the commercial components are present in the lib/karafka/pro directory of this
-      # repository and their usage requires commercial license agreement.
-      #
-      # Karafka has also commercial-friendly license, commercial support and commercial components.
-      #
-      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-      # of your code to Maciej Mensfeld.
-
       # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
       # and that allows to inject additional options into the producer, effectively allowing for a
       # much better and more granular control over the dispatch and consumption process.

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     module ActiveJob
-      # This Karafka component is a Pro component.
-      # All of the commercial components are present in the lib/karafka/pro directory of this
-      # repository and their usage requires commercial license agreement.
-      #
-      # Karafka has also commercial-friendly license, commercial support and commercial components.
-      #
-      # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-      # of your code to Maciej Mensfeld.
-
       # Contract for validating the options that can be altered with `#karafka_options` per job
       # class that works with Pro features.
       class JobOptionsContract < ::Karafka::ActiveJob::JobOptionsContract

data/lib/karafka/pro/base_consumer_extensions.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Extensions to the base consumer that make it more pro and fancy
+    #
+    # @note In case of using lrj, manual pausing may not be the best idea as resume needs to happen
+    #   after each batch is processed.
+    #
+    # They need to be added to the consumer via `#prepend`
+    module BaseConsumerExtensions
+      # Pause for tops 31 years
+      MAX_PAUSE_TIME = 1_000_000_000_000
+
+      private_constant :MAX_PAUSE_TIME
+
+      # Pauses processing of a given partition until we're done with the processing
+      # This ensures, that we can easily poll not reaching the `max.poll.interval`
+      def on_prepare
+        # Pause at the first message in a batch. That way in case of a crash, we will not loose
+        # any messages
+        pause(messages.first.offset, MAX_PAUSE_TIME) if topic.long_running_job?
+
+        super
+      end
+
+      # After user code, we seek and un-pause our partition
+      def on_consume
+        # If anything went wrong here, we should not run any partition management as it's Karafka
+        # core that will handle the backoff
+        return unless super
+
+        return unless topic.long_running_job?
+
+        # Nothing to resume if it was revoked
+        return if revoked?
+
+        # Once processing is done, we move to the new offset based on commits
+        seek(@seek_offset || messages.first.offset)
+        resume
+      end
+
+      # Marks this consumer revoked state as true
+      # This allows us for things like lrj to finish early as this state may change during lrj
+      # execution
+      def on_revoked
+        @revoked = true
+        super
+      end
+
+      # @return [Boolean] true if partition was revoked from the current consumer
+      def revoked?
+        @revoked || false
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -1,35 +1,49 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
-    # This Karafka component is a Pro component.
-    # All of the commercial components are present in the lib/karafka/pro directory of this
-    # repository and their usage requires commercial license agreement.
-    #
-    # Karafka has also commercial-friendly license, commercial support and commercial components.
-    #
-    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-    # of your code to Maciej Mensfeld.
-
     # Loader requires and loads all the pro components only when they are needed
     class Loader
+      # All the pro components that need to be loaded
+      COMPONENTS = %w[
+        performance_tracker
+        scheduler
+        base_consumer_extensions
+        processing/jobs/consume_non_blocking
+        processing/jobs_builder
+        routing/extensions
+        active_job/consumer
+        active_job/dispatcher
+        active_job/job_options_contract
+      ].freeze
+
+      private_constant :COMPONENTS
+
       class << self
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Dry::Configurable::Config] whole app config that we can alter with pro
         #   components
         def setup(config)
-          require_relative 'performance_tracker'
-          require_relative 'scheduler'
-          require_relative 'processing/jobs/consume_non_blocking'
-          require_relative 'active_job/consumer'
-          require_relative 'active_job/dispatcher'
-          require_relative 'active_job/job_options_contract'
+          COMPONENTS.each { |component| require_relative(component) }
 
           config.internal.scheduler = Scheduler.new
+          config.internal.jobs_builder = Processing::JobsBuilder.new
           config.internal.active_job.consumer = ActiveJob::Consumer
           config.internal.active_job.dispatcher = ActiveJob::Dispatcher.new
           config.internal.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
+          ::Karafka::Routing::Topic.include(Routing::Extensions)
+          ::Karafka::BaseConsumer.prepend(BaseConsumerExtensions)
+
           config.monitor.subscribe(PerformanceTracker.instance)
         end
       end

data/lib/karafka/pro/performance_tracker.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
-    # This Karafka component is a Pro component.
-    # All of the commercial components are present in the lib/karafka/pro directory of this
-    # repository and their usage requires commercial license agreement.
-    #
-    # Karafka has also commercial-friendly license, commercial support and commercial components.
-    #
-    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-    # of your code to Maciej Mensfeld.
-
     # Tracker used to keep track of performance metrics
     # It provides insights that can be used to optimize processing flow
     class PerformanceTracker

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -1,21 +1,20 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
     # Pro components related to processing part of Karafka
     module Processing
       # Pro jobs
       module Jobs
-        # This Karafka component is a Pro component.
-        # All of the commercial components are present in the lib/karafka/pro directory of this
-        # repository and their usage requires commercial license agreement.
-        #
-        # Karafka has also commercial-friendly license, commercial support and commercial
-        # components.
-        #
-        # By sending a pull request to the pro components, you are agreeing to transfer the
-        # copyright of your code to Maciej Mensfeld.
-
         # The main job type in a non-blocking variant.
         # This variant works "like" the regular consumption but pauses the partition for as long
         # as it is needed until a job is done.

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro jobs builder that supports lrj
+      class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @return [Karafka::Processing::Jobs::Consume] blocking job
+        # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
+        def consume(executor, messages)
+          if executor.topic.long_running_job?
+            Jobs::ConsumeNonBlocking.new(executor, messages)
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/extensions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions that allow to configure some extra PRO routing options
+      module Extensions
+        class << self
+          # @param base [Class] class we extend
+          def included(base)
+            base.attr_accessor :long_running_job
+          end
+        end
+
+        # @return [Boolean] is a given job on a topic a long running one
+        def long_running_job?
+          @long_running_job || false
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduler.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
 module Karafka
   module Pro
-    # This Karafka component is a Pro component.
-    # All of the commercial components are present in the lib/karafka/pro directory of this
-    # repository and their usage requires commercial license agreement.
-    #
-    # Karafka has also commercial-friendly license, commercial support and commercial components.
-    #
-    # By sending a pull request to the pro components, you are agreeing to transfer the copyright
-    # of your code to Maciej Mensfeld.
-
     # Optimizes scheduler that takes into consideration of execution time needed to process
     # messages from given topics partitions. It uses the non-preemptive LJF algorithm
     #

data/lib/karafka/processing/executor.rb

@@ -18,8 +18,15 @@ module Karafka
       # @return [String] subscription group id to which a given executor belongs
       attr_reader :group_id
 
+      # @return [Karafka::Messages::Messages] messages batch
       attr_reader :messages
 
+      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
+      # based on the topic settings defined by the end user
+      #
+      # @return [Karafka::Routing::Topic] topic of this executor
+      attr_reader :topic
+
       # @param group_id [String] id of the subscription group to which the executor belongs
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
@@ -52,7 +59,7 @@ module Karafka
           received_at
         )
 
-        consumer.on_prepared
+        consumer.on_prepare
       end
 
       # Runs consumer data processing against given batch and handles failures and errors.

data/lib/karafka/processing/jobs_builder.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Class responsible for deciding what type of job should we build to run a given command and
+    # for building a proper job for it.
+    class JobsBuilder
+      # @param executor [Karafka::Processing::Executor]
+      # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+      # @return [Karafka::Processing::Jobs::Consume] consumption job
+      def consume(executor, messages)
+        Jobs::Consume.new(executor, messages)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Revoked] revocation job
+      def revoked(executor)
+        Jobs::Revoked.new(executor)
+      end
+
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Shutdown] shutdown job
+      def shutdown(executor)
+        Jobs::Shutdown.new(executor)
+      end
+    end
+  end
+end

data/lib/karafka/processing/worker.rb

@@ -19,9 +19,13 @@ module Karafka
     class Worker
       include Helpers::Async
 
+      # @return [String] id of this worker
+      attr_reader :id
+
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
+        @id = SecureRandom.uuid
         @jobs_queue = jobs_queue
       end
 
@@ -43,19 +47,23 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          job.prepare
+          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+
+          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+            job.prepare
 
-          # If a job is marked as non blocking, we can run a tick in the job queue and if there
-          # are no other blocking factors, the job queue will be unlocked.
-          # If this does not run, all the things will be blocking and job queue won't allow to
-          # pass it until done.
-          @jobs_queue.tick(job.group_id) if job.non_blocking?
+            # If a job is marked as non blocking, we can run a tick in the job queue and if there
+            # are no other blocking factors, the job queue will be unlocked.
+            # If this does not run, all the things will be blocking and job queue won't allow to
+            # pass it until done.
+            @jobs_queue.tick(job.group_id) if job.non_blocking?
 
-          job.call
+            job.call
 
-          job.teardown
+            job.teardown
 
-          true
+            true
+          end
         else
           false
         end

data/lib/karafka/routing/subscription_group.rb

@@ -44,7 +44,7 @@ module Karafka
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
         # rdkafka offset auto-storing
-        kafka[:'enable.auto.offset.store'] = 'false'
+        kafka[:'enable.auto.offset.store'] = false
         kafka.freeze
         kafka
       end

data/lib/karafka/setup/config.rb

@@ -83,7 +83,7 @@ module Karafka
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       setting :kafka, default: {}
 
-      # Namespace for internal settings that should not be modified
+      # Namespace for internal settings that should not be modified directly
       setting :internal do
         # option routing_builder [Karafka::Routing::Builder] builder instance
         setting :routing_builder, default: Routing::Builder.new
@@ -98,6 +98,8 @@ module Karafka
         setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
         # option scheduler [Class] scheduler we will be using
         setting :scheduler, default: Scheduler.new
+        # option jobs_builder [Class] jobs builder we want to use
+        setting :jobs_builder, default: Processing::JobsBuilder.new
 
         # Karafka components for ActiveJob
         setting :active_job do

data/lib/karafka/status.rb

@@ -31,9 +31,7 @@ module Karafka
         # We skip as during this state we do not have yet a monitor
         return if initializing?
 
-        # Trap context disallows to run certain things that we instrument
-        # so the state changes are executed from a separate thread
-        Thread.new { Karafka.monitor.instrument("app.#{state}") }.join
+        Karafka.monitor.instrument("app.#{state}")
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.beta2'
+  VERSION = '2.0.0.beta3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta2
+  version: 2.0.0.beta3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-06-07 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -171,6 +171,7 @@ files:
 - bin/create_token
 - bin/integrations
 - bin/karafka
+- bin/scenario
 - bin/stress
 - certs/karafka-pro.pem
 - certs/mensfeld.pem
@@ -231,9 +232,12 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/base_consumer_extensions.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs_builder.rb
+- lib/karafka/pro/routing/extensions.rb
 - lib/karafka/pro/scheduler.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/executor.rb
@@ -242,6 +246,7 @@ files:
 - lib/karafka/processing/jobs/consume.rb
 - lib/karafka/processing/jobs/revoked.rb
 - lib/karafka/processing/jobs/shutdown.rb
+- lib/karafka/processing/jobs_builder.rb
 - lib/karafka/processing/jobs_queue.rb
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
@@ -273,6 +278,7 @@ files:
 homepage: https://karafka.io
 licenses:
 - LGPL-3.0
+- Commercial
 metadata:
   source_code_uri: https://github.com/karafka/karafka
   rubygems_mfa_required: 'true'