sidekiq 4.2.4 → 5.2.10

data/lib/sidekiq/launcher.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# encoding: utf-8
 require 'sidekiq/manager'
 require 'sidekiq/fetch'
 require 'sidekiq/scheduled'
@@ -14,6 +13,8 @@ module Sidekiq
 
     attr_accessor :manager, :poller, :fetcher
 
+    STATS_TTL = 5*365*24*60*60
+
     def initialize(options)
       @manager = Sidekiq::Manager.new(options)
       @poller = Sidekiq::Scheduled::Poller.new
@@ -39,7 +40,7 @@ module Sidekiq
     # return until all work is complete and cleaned up.
     # It can take up to the timeout to complete.
     def stop
-      deadline = Time.now + @options[:timeout]
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
 
       @done = true
       @manager.quiet
@@ -61,32 +62,36 @@ module Sidekiq
 
     private unless $TESTING
 
-    JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
-
-    def heartbeat(k, data, json)
-      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, data) }
+    def heartbeat
+      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
       results.compact!
       $0 = results.join(' ')
 
-      ❤(k, json)
+      ❤
     end
 
-    def ❤(key, json)
+    def ❤
+      key = identity
       fails = procd = 0
       begin
-        Processor::FAILURE.update {|curr| fails = curr; 0 }
-        Processor::PROCESSED.update {|curr| procd = curr; 0 }
+        fails = Processor::FAILURE.reset
+        procd = Processor::PROCESSED.reset
+        curstate = Processor::WORKER_STATE.dup
 
-        workers_key = "#{key}:workers".freeze
-        nowdate = Time.now.utc.strftime("%Y-%m-%d".freeze)
+        workers_key = "#{key}:workers"
+        nowdate = Time.now.utc.strftime("%Y-%m-%d")
         Sidekiq.redis do |conn|
           conn.multi do
-            conn.incrby("stat:processed".freeze, procd)
+            conn.incrby("stat:processed", procd)
             conn.incrby("stat:processed:#{nowdate}", procd)
-            conn.incrby("stat:failed".freeze, fails)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
             conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+
             conn.del(workers_key)
-            Processor::WORKER_STATE.each_pair do |tid, hash|
+            curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
             conn.expire(workers_key, 60)
@@ -97,8 +102,8 @@ module Sidekiq
         _, exists, _, _, msg = Sidekiq.redis do |conn|
           conn.multi do
             conn.sadd('processes', key)
-            conn.exists(key)
-            conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
+            conn.exists?(key)
+            conn.hmset(key, 'info', to_json, 'busy', curstate.size, 'beat', Time.now.to_f, 'quiet', @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           end
@@ -109,43 +114,47 @@ module Sidekiq
 
         return unless msg
 
-        if JVM_RESERVED_SIGNALS.include?(msg)
-          Sidekiq::CLI.instance.handle_signal(msg)
-        else
-          ::Process.kill(msg, $$)
-        end
+        ::Process.kill(msg, $$)
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")
         # don't lose the counts if there was a network issue
-        Processor::PROCESSED.increment(procd)
-        Processor::FAILURE.increment(fails)
+        Processor::PROCESSED.incr(procd)
+        Processor::FAILURE.incr(fails)
       end
     end
 
     def start_heartbeat
-      k = identity
-      data = {
-        'hostname' => hostname,
-        'started_at' => Time.now.to_f,
-        'pid' => $$,
-        'tag' => @options[:tag] || '',
-        'concurrency' => @options[:concurrency],
-        'queues' => @options[:queues].uniq,
-        'labels' => @options[:labels],
-        'identity' => k,
-      }
-      # this data doesn't change so dump it to a string
-      # now so we don't need to dump it every heartbeat.
-      json = Sidekiq.dump_json(data)
-
       while true
-        heartbeat(k, data, json)
+        heartbeat
         sleep 5
       end
       Sidekiq.logger.info("Heartbeat stopping...")
     end
 
+    def to_data
+      @data ||= begin
+        {
+          'hostname' => hostname,
+          'started_at' => Time.now.to_f,
+          'pid' => $$,
+          'tag' => @options[:tag] || '',
+          'concurrency' => @options[:concurrency],
+          'queues' => @options[:queues].uniq,
+          'labels' => @options[:labels],
+          'identity' => identity,
+        }
+      end
+    end
+
+    def to_json
+      @json ||= begin
+        # this data changes infrequently so dump it to a string
+        # now so we don't need to dump it every heartbeat.
+        Sidekiq.dump_json(to_data)
+      end
+    end
+
     def clear_heartbeat
       # Remove record from Redis since we are shutting down.
       # Note we don't stop the heartbeat thread; if the process

data/lib/sidekiq/logging.rb

@@ -11,7 +11,7 @@ module Sidekiq
 
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)
-        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
       end
 
       def context
@@ -22,10 +22,26 @@ module Sidekiq
 
     class WithoutTimestamp < Pretty
       def call(severity, time, program_name, message)
-        "#{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+        "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
       end
     end
 
+    def self.tid
+      Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def self.job_hash_context(job_hash)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      klass = job_hash['wrapped'] || job_hash["class"]
+      bid = job_hash['bid']
+      "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
+    end
+
+    def self.with_job_hash_context(job_hash, &block)
+      with_context(job_hash_context(job_hash), &block)
+    end
+
     def self.with_context(msg)
       Thread.current[:sidekiq_context] ||= []
       Thread.current[:sidekiq_context] << msg

data/lib/sidekiq/manager.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# encoding: utf-8
 require 'sidekiq/util'
 require 'sidekiq/processor'
 require 'sidekiq/fetch'
@@ -10,7 +9,7 @@ module Sidekiq
 
   ##
   # The Manager is the central coordination point in Sidekiq, controlling
-  # the lifecycle of the Processors and feeding them jobs as necessary.
+  # the lifecycle of the Processors.
   #
   # Tasks:
   #
@@ -31,7 +30,7 @@ module Sidekiq
     def initialize(options={})
       logger.debug { options.inspect }
       @options = options
-      @count = options[:concurrency] || 25
+      @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
@@ -54,7 +53,7 @@ module Sidekiq
 
       logger.info { "Terminating quiet workers" }
       @workers.each { |x| x.terminate }
-      fire_event(:quiet, true)
+      fire_event(:quiet, reverse: true)
     end
 
     # hack for quicker development / testing environment #2774
@@ -62,7 +61,7 @@ module Sidekiq
 
     def stop(deadline)
       quiet
-      fire_event(:shutdown, true)
+      fire_event(:shutdown, reverse: true)
 
       # some of the shutdown events can be async,
       # we don't have any way to know when they're done but
@@ -71,11 +70,11 @@ module Sidekiq
       return if @workers.empty?
 
       logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - Time.now
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       while remaining > PAUSE_TIME
         return if @workers.empty?
         sleep PAUSE_TIME
-        remaining = deadline - Time.now
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
       return if @workers.empty?
 

data/lib/sidekiq/middleware/server/active_record.rb

@@ -1,7 +1,17 @@
+# frozen_string_literal: true
 module Sidekiq
   module Middleware
     module Server
       class ActiveRecord
+
+        def initialize
+          # With Rails 5+ we must use the Reloader **always**.
+          # The reloader handles code loading and db connection management.
+          if defined?(::Rails) && defined?(::Rails::VERSION) && ::Rails::VERSION::MAJOR >= 5
+            raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
+          end
+        end
+
         def call(*args)
           yield
         ensure

data/lib/sidekiq/processor.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 require 'sidekiq/util'
 require 'sidekiq/fetch'
+require 'sidekiq/job_logger'
+require 'sidekiq/job_retry'
 require 'thread'
-require 'concurrent/map'
-require 'concurrent/atomic/atomic_fixnum'
 
 module Sidekiq
   ##
@@ -37,6 +37,8 @@ module Sidekiq
       @thread = nil
       @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
       @reloader = Sidekiq.options[:reloader]
+      @logging = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @retrier = Sidekiq::JobRetry.new
     end
 
     def terminate(wait=false)
@@ -85,7 +87,7 @@ module Sidekiq
     def get_one
       begin
         work = @strategy.retrieve_work
-        (logger.info { "Redis is online, #{Time.now - @down} sec downtime" }; @down = nil) if @down
+        (logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }; @down = nil) if @down
         work
       rescue Sidekiq::Shutdown
       rescue => ex
@@ -105,47 +107,82 @@ module Sidekiq
 
     def handle_fetch_exception(ex)
       if !@down
-        @down = Time.now
+        @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
         logger.error("Error fetching job: #{ex}")
-        ex.backtrace.each do |bt|
-          logger.error(bt)
-        end
+        handle_exception(ex)
       end
       sleep(1)
       nil
     end
 
+    def dispatch(job_hash, queue)
+      # since middleware can mutate the job hash
+      # we clone here so we report the original
+      # job structure to the Web UI
+      pristine = cloned(job_hash)
+
+      Sidekiq::Logging.with_job_hash_context(job_hash) do
+        @retrier.global(pristine, queue) do
+          @logging.call(job_hash, queue) do
+            stats(pristine, queue) do
+              # Rails 5 requires a Reloader to wrap code execution.  In order to
+              # constantize the worker and instantiate an instance, we have to call
+              # the Reloader.  It handles code loading, db connection management, etc.
+              # Effectively this block denotes a "unit of work" to Rails.
+              @reloader.call do
+                klass  = constantize(job_hash['class'])
+                worker = klass.new
+                worker.jid = job_hash['jid']
+                @retrier.local(worker, pristine, queue) do
+                  yield worker
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
     def process(work)
       jobstr = work.job
       queue = work.queue_name
 
-      ack = false
+      # Treat malformed JSON as a special case: job goes straight to the morgue.
+      job_hash = nil
       begin
-        @reloader.call do
-          job = Sidekiq.load_json(jobstr)
-          klass  = job['class'.freeze].constantize
-          worker = klass.new
-          worker.jid = job['jid'.freeze]
-
-          stats(worker, job, queue) do
-            Sidekiq.server_middleware.invoke(worker, job, queue) do
-              # Only ack if we either attempted to start this job or
-              # successfully completed it. This prevents us from
-              # losing jobs if a middleware raises an exception before yielding
-              ack = true
-              execute_job(worker, cloned(job['args'.freeze]))
-            end
+        job_hash = Sidekiq.load_json(jobstr)
+      rescue => ex
+        handle_exception(ex, { :context => "Invalid JSON for job", :jobstr => jobstr })
+        # we can't notify because the job isn't a valid hash payload.
+        DeadSet.new.kill(jobstr, notify_failure: false)
+        return work.acknowledge
+      end
+
+      ack = true
+      begin
+        dispatch(job_hash, queue) do |worker|
+          Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+            execute_job(worker, cloned(job_hash['args']))
           end
-          ack = true
         end
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
         # we didn't properly finish it.
         ack = false
+      rescue Sidekiq::JobRetry::Handled => h
+        # this is the common case: job raised error and Sidekiq::JobRetry::Handled
+        # signals that we created a retry successfully.  We can acknowlege the job.
+        e = h.cause ? h.cause : h
+        handle_exception(e, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
+        raise e
       rescue Exception => ex
-        handle_exception(ex, { :context => "Job raised exception", :job => job, :jobstr => jobstr })
-        raise
+        # Unexpected error!  This is very bad and indicates an exception that got past
+        # the retry subsystem (e.g. network partition).  We won't acknowledge the job
+        # so it can be rescued when using Sidekiq Pro.
+        ack = false
+        handle_exception(ex, { :context => "Internal exception!", :job => job_hash, :jobstr => jobstr })
+        raise e
       ensure
         work.acknowledge if ack
       end
@@ -155,34 +192,87 @@ module Sidekiq
       worker.perform(*cloned_args)
     end
 
-    def thread_identity
-      @str ||= Thread.current.object_id.to_s(36)
+    # Ruby doesn't provide atomic counters out of the box so we'll
+    # implement something simple ourselves.
+    # https://bugs.ruby-lang.org/issues/14706
+    class Counter
+      def initialize
+        @value = 0
+        @lock = Mutex.new
+      end
+
+      def incr(amount=1)
+        @lock.synchronize { @value = @value + amount }
+      end
+
+      def reset
+        @lock.synchronize { val = @value; @value = 0; val }
+      end
+    end
+
+    # jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
+    class SharedWorkerState
+      def initialize
+        @worker_state = {}
+        @lock = Mutex.new
+      end
+
+      def set(tid, hash)
+        @lock.synchronize { @worker_state[tid] = hash }
+      end
+
+      def delete(tid)
+        @lock.synchronize { @worker_state.delete(tid) }
+      end
+
+      def dup
+        @lock.synchronize { @worker_state.dup }
+      end
+
+      def size
+        @lock.synchronize { @worker_state.size }
+      end
+
+      def clear
+        @lock.synchronize { @worker_state.clear }
+      end
     end
 
-    WORKER_STATE = Concurrent::Map.new
-    PROCESSED = Concurrent::AtomicFixnum.new
-    FAILURE = Concurrent::AtomicFixnum.new
+    PROCESSED = Counter.new
+    FAILURE = Counter.new
+    WORKER_STATE = SharedWorkerState.new
 
-    def stats(worker, job, queue)
-      tid = thread_identity
-      WORKER_STATE[tid] = {:queue => queue, :payload => cloned(job), :run_at => Time.now.to_i }
+    def stats(job_hash, queue)
+      tid = Sidekiq::Logging.tid
+      WORKER_STATE.set(tid, {:queue => queue, :payload => job_hash, :run_at => Time.now.to_i })
 
       begin
         yield
       rescue Exception
-        FAILURE.increment
+        FAILURE.incr
         raise
       ensure
         WORKER_STATE.delete(tid)
-        PROCESSED.increment
+        PROCESSED.incr
       end
     end
 
     # Deep clone the arguments passed to the worker so that if
     # the job fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.
-    def cloned(ary)
-      Marshal.load(Marshal.dump(ary))
+    def cloned(thing)
+      Marshal.load(Marshal.dump(thing))
+    end
+
+    def constantize(str)
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        # the false flag limits search for name to under the constant namespace
+        #   which mimics Rails' behaviour
+        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+      end
     end
 
   end

data/lib/sidekiq/rails.rb

@@ -1,36 +1,6 @@
 # frozen_string_literal: true
-module Sidekiq
-  def self.hook_rails!
-    return if defined?(@delay_removed)
-
-    ActiveSupport.on_load(:active_record) do
-      include Sidekiq::Extensions::ActiveRecord
-    end
-
-    ActiveSupport.on_load(:action_mailer) do
-      extend Sidekiq::Extensions::ActionMailer
-    end
-
-    Module.__send__(:include, Sidekiq::Extensions::Klass)
-  end
-
-  # Removes the generic aliases which MAY clash with names of already
-  #  created methods by other applications. The methods `sidekiq_delay`,
-  #  `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
-  def self.remove_delay!
-    @delay_removed = true
-
-    [Extensions::ActiveRecord,
-     Extensions::ActionMailer,
-     Extensions::Klass].each do |mod|
-      mod.module_eval do
-        remove_method :delay if respond_to?(:delay)
-        remove_method :delay_for if respond_to?(:delay_for)
-        remove_method :delay_until if respond_to?(:delay_until)
-      end
-    end
-  end
 
+module Sidekiq
   class Rails < ::Rails::Engine
     # We need to setup this up before any application configuration which might
     # change Sidekiq middleware.
@@ -48,26 +18,16 @@ module Sidekiq
       end
     end
 
-    initializer 'sidekiq' do
-      Sidekiq.hook_rails!
-    end
-
-    # We have to add the reloader after initialize to see if cache_classes has
-    # been turned on.
-    #
-    # This hook happens after all initialziers are run, just before returning
-    # from config/environment.rb back to sidekiq/cli.rb.
     config.after_initialize do
-      if ::Rails::VERSION::MAJOR >= 5
-        # The reloader also takes care of ActiveRecord but is incompatible with
-        # the ActiveRecord middleware so make sure it's not in the chain already.
-        if defined?(Sidekiq::Middleware::Server::ActiveRecord) && Sidekiq.server_middleware.exists?(Sidekiq::Middleware::Server::ActiveRecord)
-          raise ArgumentError, "You are using the Sidekiq ActiveRecord middleware and the new Rails 5 reloader which are incompatible. Please remove the ActiveRecord middleware from your Sidekiq middleware configuration."
-        elsif ::Rails.application.config.cache_classes
-          # The reloader API has proven to be troublesome under load in production.
-          # We won't use it at all when classes are cached, see #3154
-          Sidekiq.logger.debug { "Autoload disabled in #{::Rails.env}, Sidekiq will not reload changed classes" }
-        else
+      # This hook happens after all initializers are run, just before returning
+      # from config/environment.rb back to sidekiq/cli.rb.
+      # We have to add the reloader after initialize to see if cache_classes has
+      # been turned on.
+      #
+      # None of this matters on the client-side, only within the Sidekiq process itself.
+      #
+      Sidekiq.configure_server do |_|
+        if ::Rails::VERSION::MAJOR >= 5
           Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
         end
       end
@@ -75,7 +35,6 @@ module Sidekiq
 
     class Reloader
       def initialize(app = ::Rails.application)
-        Sidekiq.logger.debug "Enabling Rails 5+ live code reloading, so hot!" unless app.config.cache_classes
         @app = app
       end
 
@@ -91,3 +50,9 @@ module Sidekiq
     end
   end if defined?(::Rails)
 end
+
+if defined?(::Rails) && ::Rails::VERSION::MAJOR < 4
+  $stderr.puts("**************************************************")
+  $stderr.puts("⛔️ WARNING: Sidekiq server is no longer supported by Rails 3.2 - please ensure your server/workers are updated")
+  $stderr.puts("**************************************************")
+end

data/lib/sidekiq/redis_connection.rb

@@ -8,11 +8,22 @@ module Sidekiq
     class << self
 
       def create(options={})
-        options = options.symbolize_keys
+        options.keys.each do |key|
+          options[key.to_sym] = options.delete(key)
+        end
 
+        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{$$}" if !options.has_key?(:id)
         options[:url] ||= determine_redis_provider
 
-        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
+        size = if options[:size]
+                 options[:size]
+               elsif Sidekiq.server?
+                 Sidekiq.options[:concurrency] + 5
+               elsif ENV['RAILS_MAX_THREADS']
+                 Integer(ENV['RAILS_MAX_THREADS'])
+               else
+                 5
+               end
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
@@ -35,7 +46,7 @@ module Sidekiq
       #   - enterprise's leader election
       #   - enterprise's cron support
       def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size <= concurrency
       end
 
       def build_client(options)
@@ -67,7 +78,14 @@ module Sidekiq
           opts.delete(:network_timeout)
         end
 
-        opts[:driver] = opts[:driver] || 'ruby'
+        opts[:driver] ||= Redis::Connection.drivers.last || 'ruby'
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
 
         opts
       end
@@ -91,7 +109,34 @@ module Sidekiq
       end
 
       def determine_redis_provider
-        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+        # If you have this in your environment:
+        # MY_REDIS_URL=redis://hostname.example.com:1238/4
+        # then set:
+        # REDIS_PROVIDER=MY_REDIS_URL
+        # and Sidekiq will find your custom URL variable with no custom
+        # initialization code at all.
+        p = ENV['REDIS_PROVIDER']
+        if p && p =~ /\:/
+          Sidekiq.logger.error <<-EOM
+
+#################################################################################
+
+REDIS_PROVIDER should be set to the **name** of the variable which contains the Redis URL, not a URL itself.
+Platforms like Heroku sell addons that publish a *_URL variable.  You tell Sidekiq with REDIS_PROVIDER, e.g.:
+
+  REDIS_PROVIDER=REDISTOGO_URL
+  REDISTOGO_URL=redis://somehost.example.com:6379/4
+
+Use REDIS_URL if you wish to point Sidekiq to a URL directly.
+
+This configuration error will crash starting in Sidekiq 5.3.
+
+#################################################################################
+EOM
+        end
+        ENV[
+          ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+        ]
       end
 
     end