sidekiq 6.0.4 → 7.2.0

data/lib/sidekiq/{worker.rb → job.rb}

@@ -4,11 +4,12 @@ require "sidekiq/client"
 
 module Sidekiq
   ##
-  # Include this module in your worker class and you can easily create
+  # Include this module in your job class and you can easily create
   # asynchronous jobs:
   #
-  #   class HardWorker
-  #     include Sidekiq::Worker
+  #   class HardJob
+  #     include Sidekiq::Job
+  #     sidekiq_options queue: 'critical', retry: 5
   #
   #     def perform(*args)
   #       # do some work
@@ -17,10 +18,30 @@ module Sidekiq
   #
   # Then in your Rails app, you can do this:
   #
-  #   HardWorker.perform_async(1, 2, 3)
+  #   HardJob.perform_async(1, 2, 3)
   #
   # Note that perform_async is a class method, perform is an instance method.
-  module Worker
+  #
+  # Sidekiq::Job also includes several APIs to provide compatibility with
+  # ActiveJob.
+  #
+  #   class SomeJob
+  #     include Sidekiq::Job
+  #     queue_as :critical
+  #
+  #     def perform(...)
+  #     end
+  #   end
+  #
+  #   SomeJob.set(wait_until: 1.hour).perform_async(123)
+  #
+  # Note that arguments passed to the job must still obey Sidekiq's
+  # best practice for simple, JSON-native data types. Sidekiq will not
+  # implement ActiveJob's more complex argument serialization. For
+  # this reason, we don't implement `perform_later` as our call semantics
+  # are very different.
+  #
+  module Job
     ##
     # The Options module is extracted so we can include it in ActiveJob::Base
     # and allow native AJs to configure Sidekiq features/internals.
@@ -36,11 +57,11 @@ module Sidekiq
         ACCESSOR_MUTEX = Mutex.new
 
         ##
-        # Allows customization for this type of Worker.
+        # Allows customization for this type of Job.
         # Legal options:
         #
         #   queue - name of queue to use for this job type, default *default*
-        #   retry - enable retries for this Worker in case of error during execution,
+        #   retry - enable retries for this Job in case of error during execution,
         #      *true* to use the default or *Integer* count
         #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
         #      can be true, false or an integer number of lines to save, default *false*
@@ -48,8 +69,8 @@ module Sidekiq
         # In practice, any option is allowed.  This is the main mechanism to configure the
         # options for a specific job.
         def sidekiq_options(opts = {})
-          opts = Hash[opts.map { |k, v| [k.to_s, v] }] # stringify
-          self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map { |k, v| [k.to_s, v] }])
+          opts = opts.transform_keys(&:to_s) # stringify
+          self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
         end
 
         def sidekiq_retry_in(&block)
@@ -61,7 +82,7 @@ module Sidekiq
         end
 
         def get_sidekiq_options # :nodoc:
-          self.sidekiq_options_hash ||= Sidekiq.default_worker_options
+          self.sidekiq_options_hash ||= Sidekiq.default_job_options
         end
 
         def sidekiq_class_attribute(*attrs)
@@ -135,7 +156,7 @@ module Sidekiq
     attr_accessor :jid
 
     def self.included(base)
-      raise ArgumentError, "Sidekiq::Worker cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
+      raise ArgumentError, "Sidekiq::Job cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
 
       base.include(Options)
       base.extend(ClassMethods)
@@ -147,49 +168,114 @@ module Sidekiq
 
     # This helper class encapsulates the set options for `set`, e.g.
     #
-    #     SomeWorker.set(queue: 'foo').perform_async(....)
+    #     SomeJob.set(queue: 'foo').perform_async(....)
     #
     class Setter
+      include Sidekiq::JobUtil
+
       def initialize(klass, opts)
         @klass = klass
-        @opts = opts
+        # NB: the internal hash always has stringified keys
+        @opts = opts.transform_keys(&:to_s)
+
+        # ActiveJob compatibility
+        interval = @opts.delete("wait_until") || @opts.delete("wait")
+        at(interval) if interval
       end
 
       def set(options)
-        @opts.merge!(options)
+        hash = options.transform_keys(&:to_s)
+        interval = hash.delete("wait_until") || @opts.delete("wait")
+        @opts.merge!(hash)
+        at(interval) if interval
         self
       end
 
       def perform_async(*args)
-        @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        if @opts["sync"] == true
+          perform_inline(*args)
+        else
+          @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        end
+      end
+
+      # Explicit inline execution of a job. Returns nil if the job did not
+      # execute, true otherwise.
+      def perform_inline(*args)
+        raw = @opts.merge("args" => args, "class" => @klass)
+
+        # validate and normalize payload
+        item = normalize_item(raw)
+        queue = item["queue"]
+
+        # run client-side middleware
+        cfg = Sidekiq.default_configuration
+        result = cfg.client_middleware.invoke(item["class"], item, queue, cfg.redis_pool) do
+          item
+        end
+        return nil unless result
+
+        # round-trip the payload via JSON
+        msg = Sidekiq.load_json(Sidekiq.dump_json(item))
+
+        # prepare the job instance
+        klass = Object.const_get(msg["class"])
+        job = klass.new
+        job.jid = msg["jid"]
+        job.bid = msg["bid"] if job.respond_to?(:bid)
+
+        # run the job through server-side middleware
+        result = cfg.server_middleware.invoke(job, msg, msg["queue"]) do
+          # perform it
+          job.perform(*msg["args"])
+          true
+        end
+        return nil unless result
+        # jobs do not return a result. they should store any
+        # modified state.
+        true
+      end
+      alias_method :perform_sync, :perform_inline
+
+      def perform_bulk(args, batch_size: 1_000)
+        client = @klass.build_client
+        client.push_bulk(@opts.merge("class" => @klass, "args" => args, :batch_size => batch_size))
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
       #   numeric (like an activesupport time interval).
       def perform_in(interval, *args)
+        at(interval).perform_async(*args)
+      end
+      alias_method :perform_at, :perform_in
+
+      private
+
+      def at(interval)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
-
-        payload = @opts.merge("class" => @klass, "args" => args)
+        ts = ((int < 1_000_000_000) ? now + int : int)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload["at"] = ts if ts > now
-        @klass.client_push(payload)
+        @opts["at"] = ts if ts > now
+        self
       end
-      alias_method :perform_at, :perform_in
     end
 
     module ClassMethods
       def delay(*args)
-        raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
+        raise ArgumentError, "Do not call .delay on a Sidekiq::Job class, call .perform_async"
       end
 
       def delay_for(*args)
-        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
+        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Job class, call .perform_in"
       end
 
       def delay_until(*args)
-        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
+        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Job class, call .perform_at"
+      end
+
+      def queue_as(q)
+        sidekiq_options("queue" => q.to_s)
       end
 
       def set(options)
@@ -197,7 +283,37 @@ module Sidekiq
       end
 
       def perform_async(*args)
-        client_push("class" => self, "args" => args)
+        Setter.new(self, {}).perform_async(*args)
+      end
+
+      # Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
+      def perform_inline(*args)
+        Setter.new(self, {}).perform_inline(*args)
+      end
+      alias_method :perform_sync, :perform_inline
+
+      ##
+      # Push a large number of jobs to Redis, while limiting the batch of
+      # each job payload to 1,000. This method helps cut down on the number
+      # of round trips to Redis, which can increase the performance of enqueueing
+      # large numbers of jobs.
+      #
+      # +items+ must be an Array of Arrays.
+      #
+      # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
+      #
+      # Example (3 Redis round trips):
+      #
+      #     SomeJob.perform_async(1)
+      #     SomeJob.perform_async(2)
+      #     SomeJob.perform_async(3)
+      #
+      # Would instead become (1 Redis round trip):
+      #
+      #     SomeJob.perform_bulk([[1], [2], [3]])
+      #
+      def perform_bulk(*args, **kwargs)
+        Setter.new(self, {}).perform_bulk(*args, **kwargs)
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -205,7 +321,7 @@ module Sidekiq
       def perform_in(interval, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => self, "args" => args}
 
@@ -217,11 +333,11 @@ module Sidekiq
       alias_method :perform_at, :perform_in
 
       ##
-      # Allows customization for this type of Worker.
+      # Allows customization for this type of Job.
       # Legal options:
       #
-      #   queue - use a named queue for this Worker, default 'default'
-      #   retry - enable the RetryJobs middleware for this Worker, *true* to use the default
+      #   queue - use a named queue for this Job, default 'default'
+      #   retry - enable the RetryJobs middleware for this Job, *true* to use the default
       #      or *Integer* count
       #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
       #      can be true, false or an integer number of lines to save, default *false*
@@ -234,13 +350,24 @@ module Sidekiq
       end
 
       def client_push(item) # :nodoc:
-        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        # stringify
-        item.keys.each do |key|
-          item[key.to_s] = item.delete(key)
+        raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
+
+        # allow the user to dynamically re-target jobs to another shard using the "pool" attribute
+        #   FooJob.set(pool: SOME_POOL).perform_async
+        old = Thread.current[:sidekiq_redis_pool]
+        pool = item.delete("pool")
+        Thread.current[:sidekiq_redis_pool] = pool if pool
+        begin
+          build_client.push(item)
+        ensure
+          Thread.current[:sidekiq_redis_pool] = old
         end
+      end
 
-        Sidekiq::Client.new(pool).push(item)
+      def build_client # :nodoc:
+        pool = Thread.current[:sidekiq_redis_pool] || get_sidekiq_options["pool"] || Sidekiq.default_configuration.redis_pool
+        client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
+        client_class.new(pool: pool)
       end
     end
   end

data/lib/sidekiq/job_logger.rb

@@ -2,7 +2,7 @@
 
 module Sidekiq
   class JobLogger
-    def initialize(logger = Sidekiq.logger)
+    def initialize(logger)
       @logger = logger
     end
 
@@ -12,46 +12,34 @@ module Sidekiq
 
       yield
 
-      with_elapsed_time_context(start) do
-        @logger.info("done")
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("done")
     rescue Exception
-      with_elapsed_time_context(start) do
-        @logger.info("fail")
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("fail")
 
       raise
     end
 
     def prepare(job_hash, &block)
-      level = job_hash["log_level"]
-      if level
-        @logger.log_at(level) do
-          Sidekiq::Context.with(job_hash_context(job_hash), &block)
-        end
-      else
-        Sidekiq::Context.with(job_hash_context(job_hash), &block)
-      end
-    end
-
-    def job_hash_context(job_hash)
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"],
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
+        jid: job_hash["jid"]
       }
-      h[:bid] = job_hash["bid"] if job_hash["bid"]
-      h[:tags] = job_hash["tags"] if job_hash["tags"]
-      h
-    end
-
-    def with_elapsed_time_context(start, &block)
-      Sidekiq::Context.with(elapsed_time_context(start), &block)
-    end
+      h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
+      h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
 
-    def elapsed_time_context(start)
-      {elapsed: elapsed(start).to_s}
+      Thread.current[:sidekiq_context] = h
+      level = job_hash["log_level"]
+      if level && @logger.respond_to?(:log_at)
+        @logger.log_at(level, &block)
+      else
+        yield
+      end
+    ensure
+      Thread.current[:sidekiq_context] = nil
     end
 
     private

data/lib/sidekiq/job_retry.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require "sidekiq/scheduled"
-require "sidekiq/api"
-
 require "zlib"
 require "base64"
+require "sidekiq/component"
 
 module Sidekiq
   ##
@@ -25,18 +23,19 @@ module Sidekiq
   #
   # A job looks like:
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
   #
   # The 'retry' option also accepts a number (in place of 'true'):
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
   #
   # The job will be retried this number of times before giving up. (If simply
   # 'true', Sidekiq retries 25 times)
   #
-  # We'll add a bit more data to the job to support retries:
+  # Relevant options for job retries:
   #
-  #  * 'queue' - the queue to use
+  #  * 'queue' - the queue for the initial job
+  #  * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
   #  * 'retry_count' - number of times we've retried so far.
   #  * 'error_message' - the message from the exception
   #  * 'error_class' - the exception class
@@ -50,30 +49,34 @@ module Sidekiq
   # The default number of retries is 25 which works out to about 3 weeks
   # You can change the default maximum number of retries in your initializer:
   #
-  #   Sidekiq.options[:max_retries] = 7
+  #   Sidekiq.default_configuration[:max_retries] = 7
   #
-  # or limit the number of retries for a particular worker with:
+  # or limit the number of retries for a particular job and send retries to
+  # a low priority queue with:
   #
-  #    class MyWorker
-  #      include Sidekiq::Worker
-  #      sidekiq_options :retry => 10
+  #    class MyJob
+  #      include Sidekiq::Job
+  #      sidekiq_options retry: 10, retry_queue: 'low'
   #    end
   #
   class JobRetry
     class Handled < ::RuntimeError; end
+
     class Skip < Handled; end
 
-    include Sidekiq::Util
+    include Sidekiq::Component
 
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
-    def initialize(options = {})
-      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    def initialize(capsule)
+      @config = @capsule = capsule
+      @max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
+      @backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
     end
 
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
-    # require the worker to be instantiated.
+    # require the job to be instantiated.
     def global(jobstr, queue)
       yield
     rescue Handled => ex
@@ -87,9 +90,9 @@ module Sidekiq
 
       msg = Sidekiq.load_json(jobstr)
       if msg["retry"]
-        attempt_retry(nil, msg, queue, e)
+        process_retry(nil, msg, queue, e)
       else
-        Sidekiq.death_handlers.each do |handler|
+        @capsule.config.death_handlers.each do |handler|
           handler.call(msg, e)
         rescue => handler_ex
           handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
@@ -100,14 +103,14 @@ module Sidekiq
     end
 
     # The local retry support means that any errors that occur within
-    # this block can be associated with the given worker instance.
+    # this block can be associated with the given job instance.
     # This is required to support the `sidekiq_retries_exhausted` block.
     #
     # Note that any exception from the block is wrapped in the Skip
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, jobstr, queue)
+    def local(jobinst, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -120,11 +123,11 @@ module Sidekiq
 
       msg = Sidekiq.load_json(jobstr)
       if msg["retry"].nil?
-        msg["retry"] = worker.class.get_sidekiq_options["retry"]
+        msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
       end
 
       raise e unless msg["retry"]
-      attempt_retry(worker, msg, queue, e)
+      process_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
       raise Skip
@@ -132,10 +135,10 @@ module Sidekiq
 
     private
 
-    # Note that +worker+ can be nil here if an error is raised before we can
-    # instantiate the worker instance.  All access must be guarded and
+    # Note that +jobinst+ can be nil here if an error is raised before we can
+    # instantiate the job instance.  All access must be guarded and
     # best effort.
-    def attempt_retry(worker, msg, queue, exception)
+    def process_retry(jobinst, msg, queue, exception)
       max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
       msg["queue"] = (msg["retry_queue"] || queue)
@@ -157,41 +160,89 @@ module Sidekiq
       end
 
       if msg["backtrace"]
+        backtrace = @backtrace_cleaner.call(exception.backtrace)
         lines = if msg["backtrace"] == true
-          exception.backtrace
+          backtrace
         else
-          exception.backtrace[0...msg["backtrace"].to_i]
+          backtrace[0...msg["backtrace"].to_i]
         end
 
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      if count < max_retry_attempts
-        delay = delay_for(worker, count, exception)
-        # Logging here can break retries if the logging device raises ENOSPC #3979
-        # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-        retry_at = Time.now.to_f + delay
-        payload = Sidekiq.dump_json(msg)
-        Sidekiq.redis do |conn|
-          conn.zadd("retry", retry_at.to_s, payload)
+      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
+
+      rf = msg["retry_for"]
+      return retries_exhausted(jobinst, msg, exception) if rf && ((msg["failed_at"] + rf) < Time.now.to_f)
+
+      strategy, delay = delay_for(jobinst, count, exception, msg)
+      case strategy
+      when :discard
+        return # poof!
+      when :kill
+        return retries_exhausted(jobinst, msg, exception)
+      end
+
+      # Logging here can break retries if the logging device raises ENOSPC #3979
+      # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+      jitter = rand(10) * (count + 1)
+      retry_at = Time.now.to_f + delay + jitter
+      payload = Sidekiq.dump_json(msg)
+      redis do |conn|
+        conn.zadd("retry", retry_at.to_s, payload)
+      end
+    end
+
+    # returns (strategy, seconds)
+    def delay_for(jobinst, count, exception, msg)
+      rv = begin
+        # sidekiq_retry_in can return two different things:
+        # 1. When to retry next, as an integer of seconds
+        # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
+        block = jobinst&.sidekiq_retry_in_block
+
+        # the sidekiq_retry_in_block can be defined in a wrapped class (ActiveJob for instance)
+        unless msg["wrapped"].nil?
+          wrapped = Object.const_get(msg["wrapped"])
+          block = wrapped.respond_to?(:sidekiq_retry_in_block) ? wrapped.sidekiq_retry_in_block : nil
         end
-      else
-        # Goodbye dear message, you (re)tried your best I'm sure.
-        retries_exhausted(worker, msg, exception)
+        block&.call(count, exception, msg)
+      rescue Exception => e
+        handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
+        nil
       end
+
+      rv = rv.to_i if rv.respond_to?(:to_i)
+      delay = (count**4) + 15
+      if Integer === rv && rv > 0
+        delay = rv
+      elsif rv == :discard
+        return [:discard, nil] # do nothing, job goes poof
+      elsif rv == :kill
+        return [:kill, nil]
+      end
+
+      [:default, delay]
     end
 
-    def retries_exhausted(worker, msg, exception)
-      begin
-        block = worker&.sidekiq_retries_exhausted_block
+    def retries_exhausted(jobinst, msg, exception)
+      rv = begin
+        block = jobinst&.sidekiq_retries_exhausted_block
+
+        # the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
+        unless msg["wrapped"].nil?
+          wrapped = Object.const_get(msg["wrapped"])
+          block = wrapped.respond_to?(:sidekiq_retries_exhausted_block) ? wrapped.sidekiq_retries_exhausted_block : nil
+        end
         block&.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      return if rv == :discard # poof!
       send_to_morgue(msg) unless msg["dead"] == false
 
-      Sidekiq.death_handlers.each do |handler|
+      @capsule.config.death_handlers.each do |handler|
         handler.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling death handler", job: msg})
@@ -201,7 +252,15 @@ module Sidekiq
     def send_to_morgue(msg)
       logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
       payload = Sidekiq.dump_json(msg)
-      DeadSet.new.kill(payload, notify_failure: false)
+      now = Time.now.to_f
+
+      redis do |conn|
+        conn.multi do |xa|
+          xa.zadd("dead", now.to_s, payload)
+          xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
+          xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
+        end
+      end
     end
 
     def retry_attempts_from(msg_retry, default)
@@ -212,26 +271,6 @@ module Sidekiq
       end
     end
 
-    def delay_for(worker, count, exception)
-      if worker&.sidekiq_retry_in_block
-        custom_retry_in = retry_in(worker, count, exception).to_i
-        return custom_retry_in if custom_retry_in > 0
-      end
-      seconds_to_delay(count)
-    end
-
-    # delayed_job uses the same basic formula
-    def seconds_to_delay(count)
-      (count**4) + 15 + (rand(30) * (count + 1))
-    end
-
-    def retry_in(worker, count, exception)
-      worker.sidekiq_retry_in_block.call(count, exception)
-    rescue Exception => e
-      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
-      nil
-    end
-
     def exception_caused_by_shutdown?(e, checked_causes = [])
       return false unless e.cause