workhorse 1.5.2 → 2.0.0.rc0

data/lib/workhorse/notifiers/redis.rb

@@ -0,0 +1,227 @@
+module Workhorse
+  module Notifiers
+    # Notifier that announces an enqueued job over a Redis pub/sub channel,
+    # for deployments whose workers do not share a filesystem with the
+    # application.
+    #
+    # Redis is a soft dependency: it is not declared in the gemspec and is only
+    # required when this notifier is selected. Configure the client and,
+    # optionally, the channel:
+    #
+    # ```ruby
+    # Workhorse.setup do |config|
+    #   config.notifier = :redis
+    #   config.notification_redis = Redis.new(url: ENV['REDIS_URL'])
+    # end
+    # ```
+    #
+    # A subscriber thread per worker process keeps a counter, which pollers
+    # read through {#token}. Delivery is best-effort in both directions: a
+    # publish that fails and a subscription that drops both cost latency only,
+    # as polling still picks the job up.
+    class Redis < Base
+      # Default channel jobs are announced on.
+      DEFAULT_CHANNEL = 'workhorse:jobs'.freeze
+
+      # Seconds to wait before subscribing again after a failure.
+      RECONNECT_DELAY = 1
+
+      # @param client [Object, nil] Redis client. Defaults to
+      #   {Workhorse.notification_redis}.
+      # @param channel [String, nil] Channel to use. Defaults to
+      #   {Workhorse.notification_channel} or {DEFAULT_CHANNEL}.
+      def initialize(client: nil, channel: nil)
+        super()
+        @configured_client = client
+        @channel = channel
+        @counter = Concurrent::AtomicFixnum.new(0)
+        @subscribers = 0
+        @mutex = Mutex.new
+      end
+
+      # Resolved when used rather than when constructed, so that
+      # {Workhorse.notification_channel} can be set in any order relative to
+      # {Workhorse.notifier=} - up to the point a worker starts, as the
+      # subscriber thread captures the channel it was started with.
+      #
+      # @return [String] Channel that is published to and subscribed on
+      def channel
+        return @channel || Workhorse.notification_channel || DEFAULT_CHANNEL
+      end
+
+      # @return [Object] The configured Redis client
+      # @raise [RuntimeError] If no client has been configured
+      def client
+        return @client if @client
+
+        # Built under the mutex: #notify runs on every enqueueing thread, and
+        # two of them racing here would each build one and orphan the loser.
+        @mutex.synchronize { @client ||= build_client }
+
+        return @client
+      end
+
+      # Returns the configured source of clients: either a client, or a
+      # callable returning one.
+      #
+      # @return [Object]
+      # @raise [RuntimeError] If nothing has been configured
+      # @private
+      def client_source
+        return @configured_client || Workhorse.notification_redis \
+          || fail('Workhorse.notification_redis must be set to a Redis client, or to something ' \
+                  'callable returning one, to use the :redis notifier.')
+      end
+
+      # Publishes the queue name on the configured channel.
+      #
+      # @param queue [String, Symbol, nil] Queue the job was enqueued into
+      # @return [void]
+      def notify(queue: nil)
+        client.publish(channel, queue.to_s)
+      rescue StandardError => e
+        # Best-effort by contract, see Workhorse::Notifiers::Base#notify.
+        Workhorse.debug_log("Notification failed: #{e.class}: #{e.message}")
+      end
+
+      # Starts the subscriber thread, unless one is already running for this
+      # process.
+      #
+      # @return [void]
+      def start
+        @mutex.synchronize do
+          @subscribers += 1
+          @thread = start_subscriber unless @thread&.alive?
+        end
+
+        return
+      end
+
+      # Stops the subscriber thread once the last worker in this process has
+      # stopped.
+      #
+      # @return [void]
+      def stop
+        @mutex.synchronize do
+          @subscribers -= 1 if @subscribers > 0
+          next unless @subscribers.zero?
+
+          @thread&.kill
+          @thread = nil
+          # The thread held the only reference to it, so nothing else would
+          # ever close it.
+          @subscriber_client = close(@subscriber_client)
+          @subscriber_failed = false
+        end
+
+        return
+      end
+
+      # @return [Integer] Number of notifications received in this process
+      def token
+        return @counter.value
+      end
+
+      private
+
+      # Subscribes on a thread of its own, reconnecting after a dropped
+      # connection.
+      #
+      # @return [Thread]
+      def start_subscriber
+        counter = @counter
+        chan = channel
+
+        return Thread.new do
+          loop do
+            # Kept across iterations and replaced only once the connection it
+            # holds has failed, so an outage does not build one client per
+            # second and drop each unclosed.
+            client = (@subscriber_client ||= subscriber_client)
+
+            client.subscribe(chan) do |on|
+              # Re-armed when the subscription is established, not after
+              # #subscribe returns: it blocks for the subscription's whole
+              # life, so a later outage would otherwise never be reported.
+              on.subscribe { @subscriber_failed = false }
+              on.message { |_channel, _message| counter.increment }
+            end
+          rescue StandardError => e
+            report_subscriber_failure(e)
+            @subscriber_client = close(client)
+            Kernel.sleep RECONNECT_DELAY
+          end
+        end
+      end
+
+      # Reports a failed subscription. Retrying is silent after the first
+      # report: a broker that is down produces one of these per
+      # {RECONNECT_DELAY}, and a misconfigured notifier would otherwise retry
+      # forever without anyone hearing about it.
+      #
+      # @param exception [Exception]
+      # @return [void]
+      def report_subscriber_failure(exception)
+        message = "Notification subscriber failed: #{exception.class}: #{exception.message}"
+        Workhorse.debug_log(message)
+
+        return if @subscriber_failed
+
+        @subscriber_failed = true
+
+        begin
+          Workhorse.on_exception.call(exception)
+        rescue Exception => e
+          # A reporter that is itself unreachable must not unwind the retry
+          # loop and leave the process without a subscriber for good.
+          Workhorse.debug_log("on_exception failed: #{e.class}: #{e.message}")
+        end
+
+        return
+      end
+
+      # Closes a client this notifier built, tolerating one that cannot be
+      # closed. A client supplied by the application is left alone: it may be
+      # a wrapper sharing its connection with the one {#notify} publishes on,
+      # which closing would take down.
+      #
+      # @param client [Object, nil]
+      # @return [nil]
+      def close(client)
+        client.close if @owns_subscriber_client && client.respond_to?(:close)
+
+        return nil
+      rescue StandardError => e
+        Workhorse.debug_log("Closing the subscriber client failed: #{e.class}: #{e.message}")
+
+        return nil
+      end
+
+      # @return [Object] A newly built client
+      def build_client
+        source = client_source
+
+        return source.respond_to?(:call) ? source.call : source
+      end
+
+      # Returns the client to subscribe with. A subscribed connection cannot
+      # be used for anything else, so this must not be the client that
+      # {#notify} publishes on.
+      #
+      # Configure {Workhorse.notification_redis} with something callable to
+      # get a connection of its own here. Given a client instead, this falls
+      # back to `dup`. redis-rb's own `dup` builds a fresh client, but a
+      # wrapper that does not define one - a namespacing or instrumenting
+      # delegator - yields a shallow copy sharing the connection, which is
+      # why such a client is never closed.
+      #
+      # @return [Object]
+      def subscriber_client
+        source = client_source
+        @owns_subscriber_client = source.respond_to?(:call)
+
+        return @owns_subscriber_client ? source.call : source.dup
+      end
+    end
+  end
+end

data/lib/workhorse/performer.rb

@@ -73,6 +73,32 @@ module Workhorse
       end
     end
 
+    # Calls {Workhorse.on_job_late} if the job started later than its
+    # `max_lateness` allows.
+    #
+    # Unlike an expiry, this is not a reason to skip the job: it ran, just
+    # not on time. A failing callback must not fail the job either, so it is
+    # reported through {Workhorse.on_exception} instead.
+    #
+    # @return [void]
+    # @private
+    def report_lateness
+      return unless @db_job.has_attribute?(:max_lateness)
+      return unless @db_job.max_lateness
+
+      lateness = @db_job.lateness
+
+      return if lateness.nil? || lateness <= @db_job.max_lateness
+
+      log "Started #{lateness.round(1)}s after the intended #{@db_job.perform_at}, " \
+          "which exceeds the configured maximum of #{@db_job.max_lateness}s", :warn
+
+      Workhorse.on_job_late.call(@db_job, lateness)
+    rescue Exception => e
+      log %(on_job_late failed: #{e.message}), :error
+      Workhorse.on_exception.call(e)
+    end
+
     # Core job execution logic with state transitions.
     # Handles marking job as started, deserializing and executing the job,
     # and marking as succeeded.
@@ -89,6 +115,8 @@ module Workhorse
         @db_job.mark_started!
       end
 
+      report_lateness
+
       # ---------------------------------------------------------------
       # Deserialize and perform job
       # ---------------------------------------------------------------