sidekiq 7.3.9 → 8.1.7

data/lib/sidekiq/cli.rb

@@ -2,8 +2,6 @@
 
 $stdout.sync = true
 
-require "yaml"
-require "singleton"
 require "optparse"
 require "erb"
 require "fileutils"
@@ -17,7 +15,6 @@ require "sidekiq/launcher"
 module Sidekiq # :nodoc:
   class CLI
     include Sidekiq::Component
-    include Singleton unless $TESTING
 
     attr_accessor :launcher
     attr_accessor :environment
@@ -31,6 +28,10 @@ module Sidekiq # :nodoc:
       validate!
     end
 
+    def self.instance
+      @instance ||= new
+    end
+
     def jruby?
       defined?(::JRUBY_VERSION)
     end
@@ -47,7 +48,7 @@ module Sidekiq # :nodoc:
       logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
-      sigs = %w[INT TERM TTIN TSTP]
+      sigs = %w[INT TERM INFO TTIN TSTP]
       # USR1 and USR2 don't work on the JVM
       sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
@@ -74,7 +75,7 @@ module Sidekiq # :nodoc:
       # fire startup and start multithreading.
       info = @config.redis_info
       ver = Gem::Version.new(info["redis_version"])
-      raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
+      raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
 
       maxmemory_policy = info["maxmemory_policy"]
       if maxmemory_policy != "noeviction" && maxmemory_policy != ""
@@ -208,14 +209,25 @@ module Sidekiq # :nodoc:
             cli.logger.warn "<no backtrace available>"
           end
         end
+      },
+      "INFO" => ->(cli) {
+        cli.logger.error { "DEPRECATED: the INFO signal does not work on Linux, use TTIN instead." }
+        Thread.list.each do |thread|
+          cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+          if thread.backtrace
+            cli.logger.warn thread.backtrace.join("\n")
+          else
+            cli.logger.warn "<no backtrace available>"
+          end
+        end
       }
     }
-    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
-    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       logger.debug "Got #{sig} signal"
-      SIGNAL_HANDLERS[sig].call(self)
+      hndlr = SIGNAL_HANDLERS[sig]
+      hndlr ? hndlr.call(self) :
+        logger.warn("No #{sig} signal handler registered, ignoring")
     end
 
     private
@@ -298,29 +310,18 @@ module Sidekiq # :nodoc:
 
       if File.directory?(@config[:require])
         require "rails"
-        if ::Rails::VERSION::MAJOR < 6
-          warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
+        if ::Rails::VERSION::MAJOR < 7
+          warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 7+"
         end
         require "sidekiq/rails"
         require File.expand_path("#{@config[:require]}/config/environment.rb")
-        @config[:tag] ||= default_tag
+        @config[:tag] ||= default_tag(::Rails.root)
       else
         require @config[:require]
+        @config[:tag] ||= default_tag
       end
     end
 
-    def default_tag
-      dir = ::Rails.root
-      name = File.basename(dir)
-      prevdir = File.dirname(dir) # Capistrano release directory?
-      if name.to_i != 0 && prevdir
-        if File.basename(prevdir) == "releases"
-          return File.basename(File.dirname(prevdir))
-        end
-      end
-      name
-    end
-
     def validate!
       if !File.exist?(@config[:require]) ||
           (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
@@ -395,12 +396,18 @@ module Sidekiq # :nodoc:
     end
 
     def initialize_logger
-      @config.logger.level = ::Logger::DEBUG if @config[:verbose]
+      if @config[:verbose] || ENV["DEBUG_INVOCATION"] == "1"
+        # DEBUG_INVOCATION is a systemd-ism triggered by
+        # RestartMode=debug. We turn on debugging when the
+        # sidekiq process crashes and is restarted with this flag.
+        @config.logger.level = ::Logger::DEBUG
+      end
     end
 
     def parse_config(path)
       erb = ERB.new(File.read(path), trim_mode: "-")
       erb.filename = File.expand_path(path)
+      require "yaml"
       opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
 
       if opts.respond_to? :deep_symbolize_keys!

data/lib/sidekiq/client.rb

@@ -67,9 +67,7 @@ module Sidekiq
         c.pipelined do |p|
           p.hsetnx(key, "cancelled", Time.now.to_i)
           p.hget(key, "cancelled")
-          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
-          # TODO When Redis 7.2 is required
-          # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
         end
       end
       result.to_i
@@ -119,6 +117,14 @@ module Sidekiq
     # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
+    # Accepts an `:at` option to schedule the jobs for future execution. It
+    # accepts either a single Numeric timestamp (or seconds-from-now) applied
+    # to every job, or an Array of Numeric values with the same size as `args`
+    # to schedule each job at its corresponding time.
+    #
+    # Accepts an additional `:spread_interval` option (in seconds) to randomly spread
+    # the jobs schedule times over the specified interval.
+    #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
     # is run through the client middleware pipeline and each job gets its own Job ID
@@ -131,15 +137,28 @@ module Sidekiq
     #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
     #
     def push_bulk(items)
-      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
-      at = items.delete("at")
+      at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
+      spread_interval = items.delete(:spread_interval) || items.delete("spread_interval")
+      raise ArgumentError, "Jobs 'spread_interval' must be a positive Numeric" if spread_interval && (!spread_interval.is_a?(Numeric) || spread_interval <= 0)
+      raise ArgumentError, "Only one of 'at' or 'spread_interval' can be provided" if at && spread_interval
+
+      if !at && spread_interval
+        # Do not use spread interval smaller than pooling interval.
+        spread_interval = [spread_interval, 5].max
+        now = Time.now.to_f
+        at = args.map { now + rand * spread_interval }
+      end
+
+      # Use a smaller batch size by default for scheduled jobs since adding to sorted sets is more costly.
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || (at ? 100 : 1_000)
+
       normed = normalize_item(items)
       slice_index = 0
       result = args.each_slice(batch_size).flat_map do |slice|
@@ -266,22 +285,21 @@ module Sidekiq
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash["at"].to_s
-          # ActiveJob sets this but the job has not been enqueued yet
-          hash.delete("enqueued_at")
-          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
-          hash = hash.dup
-          hash.delete("at")
+          # ActiveJob sets enqueued_at but the job has not been enqueued yet
+          hash = hash.except("enqueued_at", "at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
-        queue = payloads.first["queue"]
-        now = Time.now.to_f
-        to_push = payloads.map { |entry|
-          entry["enqueued_at"] = now
-          Sidekiq.dump_json(entry)
-        }
-        conn.sadd("queues", [queue])
-        conn.lpush("queue:#{queue}", to_push)
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) # milliseconds since the epoch
+        grouped_queues = payloads.group_by { |job| job["queue"] }
+        conn.sadd("queues", grouped_queues.keys)
+        grouped_queues.each do |queue, grouped_payloads|
+          to_push = grouped_payloads.map { |entry|
+            entry["enqueued_at"] = now
+            Sidekiq.dump_json(entry)
+          }
+          conn.lpush("queue:#{queue}", to_push)
+        end
       end
     end
   end

data/lib/sidekiq/component.rb

@@ -1,11 +1,39 @@
 # frozen_string_literal: true
 
 module Sidekiq
+  # Ruby's default thread priority is 0, which uses 100ms time slices.
+  # This can lead to some surprising thread starvation; if using a lot of
+  # CPU-heavy concurrency, it may take several seconds before a Thread gets
+  # on the CPU.
+  #
+  # Negative priorities lower the timeslice by half, so -1 = 50ms, -2 = 25ms, etc.
+  # With more frequent timeslices, we reduce the risk of unintentional timeouts
+  # and starvation.
+  #
+  # Customize like so:
+  #
+  #   Sidekiq.configure_server do |cfg|
+  #     cfg.thread_priority = 0
+  #   end
+  #
+  DEFAULT_THREAD_PRIORITY = -1
+
   ##
-  # Sidekiq::Component assumes a config instance is available at @config
+  # Sidekiq::Component provides a set of utility methods depending only
+  # on Sidekiq::Config. It assumes a config instance is available at @config.
   module Component # :nodoc:
     attr_reader :config
 
+    # This is epoch milliseconds, appropriate for persistence
+    def real_ms
+      ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+    end
+
+    # used for time difference and relative comparisons, not persistence.
+    def mono_ms
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex
@@ -13,11 +41,11 @@ module Sidekiq
       raise ex
     end
 
-    def safe_thread(name, &block)
+    def safe_thread(name, priority: nil, &block)
       Thread.new do
         Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
-      end
+      end.tap { |t| t.priority = (priority || config.thread_priority || DEFAULT_THREAD_PRIORITY) }
     end
 
     def logger
@@ -29,6 +57,9 @@ module Sidekiq
     end
 
     def tid
+      # We XOR with PID to ensure Thread IDs changes after fork.
+      # I'm unclear why we don't multiply the two values to better guarantee
+      # a unique value but it's been this way for quite a while now. #3685
       Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
     end
 
@@ -86,5 +117,16 @@ module Sidekiq
         end.join(", ")
       }>"
     end
+
+    def default_tag(dir = Dir.pwd)
+      name = File.basename(dir)
+      prevdir = File.dirname(dir) # Capistrano release directory?
+      if name.to_i != 0 && prevdir
+        if File.basename(prevdir) == "releases"
+          return File.basename(File.dirname(prevdir))
+        end
+      end
+      name
+    end
   end
 end

data/lib/sidekiq/config.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "forwardable"
-
-require "set"
 require "sidekiq/redis_connection"
 
 module Sidekiq
@@ -19,16 +17,16 @@ module Sidekiq
       poll_interval_average: nil,
       average_scheduled_poll_interval: 5,
       on_complex_arguments: :raise,
-      iteration: {
-        max_job_runtime: nil,
-        retry_backoff: 0
-      },
+      # if the Iterable job runs longer than this value (in seconds), then the job
+      # will be interrupted after the current iteration and re-enqueued at the back of the queue
+      max_iteration_runtime: nil,
       error_handlers: [],
       death_handlers: [],
       lifecycle_events: {
         startup: [],
         quiet: [],
         shutdown: [],
+        exit: [],
         # triggers when we fire the first heartbeat on startup OR repairing a network partition
         heartbeat: [],
         # triggers on EVERY heartbeat call, every 10 seconds
@@ -37,16 +35,28 @@ module Sidekiq
       dead_max_jobs: 10_000,
       dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
       reloader: proc { |&block| block.call },
-      backtrace_cleaner: ->(backtrace) { backtrace }
+      backtrace_cleaner: ->(backtrace) { backtrace },
+      logged_job_attributes: ["bid", "tags"],
+      redis_idle_timeout: nil
     }
 
     ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
-      l = cfg.logger
-      l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
-      l.warn("#{ex.class.name}: #{ex.message}")
-      unless ex.backtrace.nil?
-        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
-        l.warn(backtrace.join("\n"))
+      Sidekiq::Context.with(ctx) do
+        dev = cfg[:environment] == "development"
+        fancy = dev && $stdout.tty? # 🎩
+        # Weird logic here but we want to show the backtrace in local
+        # development or if verbose logging is enabled.
+        #
+        # `full_message` contains the error class, message and backtrace
+        # `detailed_message` contains the error class and message
+        #
+        # Absolutely terrible API names. Not useful at all to have two
+        # methods with similar but obscure names.
+        if dev || cfg.logger.debug?
+          cfg.logger.info { ex.full_message(highlight: fancy) }
+        else
+          cfg.logger.info { ex.detailed_message(highlight: fancy) }
+        end
       end
     }
 
@@ -60,6 +70,7 @@ module Sidekiq
 
     def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
+    attr_accessor :thread_priority
 
     def inspect
       "#<#{self.class.name} @options=#{
@@ -135,11 +146,15 @@ module Sidekiq
       @redis_config = @redis_config.merge(hash)
     end
 
+    def reap_idle_redis_connections(timeout = 60)
+      self[:redis_idle_timeout] = timeout
+    end
+
     def redis_pool
       Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
     end
 
-    private def local_redis_pool
+    def local_redis_pool
       # this is our internal client/housekeeping pool. each capsule has its
       # own pool for executing threads.
       @redis ||= new_redis_pool(10, "internal")
@@ -240,7 +255,7 @@ module Sidekiq
     # Register a proc to handle any error which occurs within the Sidekiq process.
     #
     #   Sidekiq.configure_server do |config|
-    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #     config.error_handlers << proc {|ex,ctx_hash,config| MyErrorService.notify(ex, ctx_hash) }
     #   end
     #
     # The default error handler logs errors to @logger.
@@ -249,7 +264,7 @@ module Sidekiq
     end
 
     # Register a block to run at a point in the Sidekiq lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
+    # :startup, :quiet, :shutdown, or :exit are valid events.
     #
     #   Sidekiq.configure_server do |config|
     #     config.on(:shutdown) do
@@ -282,24 +297,14 @@ module Sidekiq
       @logger = logger
     end
 
-    private def parameter_size(handler)
-      target = handler.is_a?(Proc) ? handler : handler.method(:call)
-      target.parameters.size
-    end
-
     # INTERNAL USE ONLY
     def handle_exception(ex, ctx = {})
-      if @options[:error_handlers].size == 0
-        p ["!!!!!", ex]
+      if @options[:error_handlers].empty?
+        logger.error { "No error handlers configured, logging exception directly" }
+        logger.error { ex }
       end
       @options[:error_handlers].each do |handler|
-        if parameter_size(handler) == 2
-          # TODO Remove in 8.0
-          logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
-          handler.call(ex, {_config: self}.merge(ctx))
-        else
-          handler.call(ex, ctx, self)
-        end
+        handler.call(ex, ctx, self)
       rescue Exception => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"

data/lib/sidekiq/embedded.rb

@@ -34,6 +34,7 @@ module Sidekiq
     private
 
     def housekeeping
+      @config[:tag] ||= default_tag
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
       logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
@@ -42,7 +43,7 @@ module Sidekiq
       # fire startup and start multithreading.
       info = config.redis_info
       ver = Gem::Version.new(info["redis_version"])
-      raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
+      raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
 
       maxmemory_policy = info["maxmemory_policy"]
       if maxmemory_policy != "noeviction"

data/lib/sidekiq/fetch.rb

@@ -7,6 +7,7 @@ require "sidekiq/capsule"
 module Sidekiq # :nodoc:
   class BasicFetch
     include Sidekiq::Component
+
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2

data/lib/sidekiq/iterable_job.rb

@@ -46,6 +46,7 @@ module Sidekiq
     # def on_start
     # def on_resume
     # def on_stop
+    # def on_cancel
     # def on_complete
     # def around_iteration
     #

data/lib/sidekiq/job/iterable.rb

@@ -32,8 +32,14 @@ module Sidekiq
         @_runtime = 0
         @_args = nil
         @_cancelled = nil
+        @current_object = nil
       end
 
+      # Access to the current object while iterating.
+      # This value is not reset so the latest element is
+      # explicitly available to cleanup/complete callbacks.
+      attr_reader :current_object
+
       def arguments
         @_args
       end
@@ -49,14 +55,12 @@ module Sidekiq
       def cancel!
         return @_cancelled if cancelled?
 
-        key = "it-#{jid}"
+        key = iteration_key
         _, result, _ = Sidekiq.redis do |c|
           c.pipelined do |p|
             p.hsetnx(key, "cancelled", Time.now.to_i)
             p.hget(key, "cancelled")
-            # TODO When Redis 7.2 is required
-            # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
-            p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
+            p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
           end
         end
         @_cancelled = result.to_i
@@ -66,6 +70,10 @@ module Sidekiq
         @_cancelled
       end
 
+      def cursor
+        @_cursor.freeze
+      end
+
       # A hook to override that will be called when the job starts iterating.
       #
       # It is called only once, for the first time.
@@ -93,6 +101,11 @@ module Sidekiq
       def on_stop
       end
 
+      # A hook to override that will be called when the job is cancelled.
+      #
+      def on_cancel
+      end
+
       # A hook to override that will be called when the job finished iterating.
       #
       def on_complete
@@ -130,7 +143,7 @@ module Sidekiq
         fetch_previous_iteration_state
 
         @_executions += 1
-        @_start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        @_start_time = mono_now
 
         enumerator = build_enumerator(*args, cursor: @_cursor)
         unless enumerator
@@ -164,7 +177,7 @@ module Sidekiq
       private
 
       def is_cancelled?
-        @_cancelled = Sidekiq.redis { |c| c.hget("it-#{jid}", "cancelled") }
+        @_cancelled = Sidekiq.redis { |c| c.hget(iteration_key, "cancelled") }
       end
 
       def fetch_previous_iteration_state
@@ -184,34 +197,40 @@ module Sidekiq
 
       def iterate_with_enumerator(enumerator, arguments)
         if is_cancelled?
+          on_cancel
           logger.info { "Job cancelled" }
           return true
         end
 
         time_limit = Sidekiq.default_configuration[:timeout]
         found_record = false
-        state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        state_flushed_at = mono_now
 
         enumerator.each do |object, cursor|
           found_record = true
           @_cursor = cursor
+          @current_object = object
 
-          is_interrupted = interrupted?
-          if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - state_flushed_at >= STATE_FLUSH_INTERVAL || is_interrupted
+          interrupt_job = interrupted? || should_interrupt?
+          if mono_now - state_flushed_at >= STATE_FLUSH_INTERVAL || interrupt_job
             _, _, cancelled = flush_state
-            state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+            state_flushed_at = mono_now
             if cancelled
               @_cancelled = true
+              on_cancel
               logger.info { "Job cancelled" }
               return true
             end
           end
 
-          return false if is_interrupted
+          return false if interrupt_job
 
-          verify_iteration_time(time_limit, object) do
+          verify_iteration_time(time_limit) do
             around_iteration do
               each_iteration(object, *arguments)
+            rescue Exception
+              flush_state
+              raise
             end
           end
         end
@@ -219,16 +238,16 @@ module Sidekiq
         logger.debug("Enumerator found nothing to iterate!") unless found_record
         true
       ensure
-        @_runtime += (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @_start_time)
+        @_runtime += (mono_now - @_start_time)
       end
 
-      def verify_iteration_time(time_limit, object)
-        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      def verify_iteration_time(time_limit)
+        start = mono_now
         yield
-        finish = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        finish = mono_now
         total = finish - start
         if total > time_limit
-          logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d) when processing `%s`. This can lead to job processing problems during deploys" % [total, time_limit, object] }
+          logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d). This can lead to job processing problems during deploys" % [total, time_limit] }
         end
       end
 
@@ -254,6 +273,11 @@ module Sidekiq
         end
       end
 
+      def should_interrupt?
+        max_iteration_runtime = Sidekiq.default_configuration[:max_iteration_runtime]
+        max_iteration_runtime && (mono_now - @_start_time > max_iteration_runtime)
+      end
+
       def flush_state
         key = iteration_key
         state = {
@@ -265,7 +289,7 @@ module Sidekiq
         Sidekiq.redis do |conn|
           conn.multi do |pipe|
             pipe.hset(key, state)
-            pipe.expire(key, STATE_TTL)
+            pipe.expire(key, STATE_TTL, "nx")
             pipe.hget(key, "cancelled")
           end
         end
@@ -289,6 +313,10 @@ module Sidekiq
           raise "Unexpected thrown value: #{completed.inspect}"
         end
       end
+
+      def mono_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end