sidekiq-cron 0.6.3 → 1.10.0

data/lib/sidekiq/cron/job.rb

@@ -1,42 +1,45 @@
+require 'fugit'
+require 'globalid'
 require 'sidekiq'
-require 'sidekiq/util'
-require 'rufus-scheduler'
 require 'sidekiq/cron/support'
+require 'sidekiq/options'
 
 module Sidekiq
   module Cron
-
     class Job
-      include Util
-      extend Util
-
-      #how long we would like to store informations about previous enqueues
+      # How long we would like to store informations about previous enqueues.
       REMEMBER_THRESHOLD = 24 * 60 * 60
 
-      #crucial part of whole enquing job
+      # Time format for enqueued jobs.
+      LAST_ENQUEUE_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %z'
+
+      # Use the exists? method if we're on a newer version of Redis.
+      REDIS_EXISTS_METHOD = Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("7.0.0") || Gem.loaded_specs['redis'].version < Gem::Version.new('4.2') ? :exists : :exists?
+
+      # Use serialize/deserialize key of GlobalID.
+      GLOBALID_KEY = "_sc_globalid"
+
+      # Crucial part of whole enqueuing job.
       def should_enque? time
-        enqueue = false
         enqueue = Sidekiq.redis do |conn|
           status == "enabled" &&
             not_past_scheduled_time?(time) &&
             not_enqueued_after?(time) &&
-            conn.zadd(job_enqueued_key, formated_enqueue_time(time), formated_last_time(time))
+            conn.zadd(job_enqueued_key, formatted_enqueue_time(time), formatted_last_time(time))
         end
-        enqueue
+        enqueue == true || enqueue == 1
       end
 
-      # remove previous informations about run times
-      # this will clear redis and make sure that redis will
-      # not overflow with memory
+      # Remove previous information about run times,
+      # this will clear Redis and make sure that Redis will not overflow with memory.
       def remove_previous_enques time
         Sidekiq.redis do |conn|
           conn.zremrangebyscore(job_enqueued_key, 0, "(#{(time.to_f - REMEMBER_THRESHOLD).to_s}")
         end
       end
 
-      #test if job should be enqued If yes add it to queue
+      # Test if job should be enqueued.
       def test_and_enque_for_time! time
-        #should this job be enqued?
         if should_enque?(time)
           enque!
 
@@ -44,7 +47,7 @@ module Sidekiq
         end
       end
 
-      #enque cron job to queue
+      # Enqueue cron job to queue.
       def enque! time = Time.now.utc
         @last_enqueue_time = time
 
@@ -55,45 +58,54 @@ module Sidekiq
               nil
             end
 
-        if klass_const
-          if defined?(ActiveJob::Base) && klass_const < ActiveJob::Base
-            enqueue_active_job(klass_const)
-          else
-            enqueue_sidekiq_worker(klass_const)
-          end
-        else
-          if @active_job
-            Sidekiq::Client.push(active_job_message)
+        jid =
+          if klass_const
+            if is_active_job?(klass_const)
+              enqueue_active_job(klass_const).try :provider_job_id
+            else
+              enqueue_sidekiq_worker(klass_const)
+            end
           else
-            Sidekiq::Client.push(sidekiq_worker_message)
+            if @active_job
+              Sidekiq::Client.push(active_job_message)
+            else
+              Sidekiq::Client.push(sidekiq_worker_message)
+            end
           end
-        end
 
         save_last_enqueue_time
-        logger.debug { "enqueued #{@name}: #{@message}" }
+        add_jid_history jid
+        Sidekiq.logger.debug { "enqueued #{@name}: #{@message}" }
       end
 
-      def is_active_job?
-        @active_job || defined?(ActiveJob::Base) && Sidekiq::Cron::Support.constantize(@klass.to_s) < ActiveJob::Base
+      def is_active_job?(klass = nil)
+        @active_job || defined?(ActiveJob::Base) && (klass || Sidekiq::Cron::Support.constantize(@klass.to_s)) < ActiveJob::Base
       rescue NameError
         false
       end
 
-      def enqueue_active_job(klass_const)
-        klass_const.set(queue: @queue).perform_later(*@args)
+      def date_as_argument?
+        !!@date_as_argument
+      end
 
-        true
+      def enqueue_args
+        args = date_as_argument? ? @args + [Time.now.to_f] : @args
+        deserialize_argument(args)
       end
 
-      def enqueue_sidekiq_worker(klass_const)
-        klass_const.set(queue: queue_name_with_prefix).perform_async(*@args)
+      def enqueue_active_job(klass_const)
+        klass_const.set(queue: @queue).perform_later(*enqueue_args)
+      end
 
-        true
+      def enqueue_sidekiq_worker(klass_const)
+        klass_const.set(queue: queue_name_with_prefix).perform_async(*enqueue_args)
       end
 
-      # siodekiq worker message
+      # Sidekiq worker message.
       def sidekiq_worker_message
-        @message.is_a?(String) ? Sidekiq.load_json(@message) : @message
+        message = @message.is_a?(String) ? Sidekiq.load_json(@message) : @message
+        message["args"] = enqueue_args
+        message
       end
 
       def queue_name_with_prefix
@@ -118,24 +130,25 @@ module Sidekiq
         queue_name
       end
 
-      # active job has different structure how it is loading data from sidekiq
-      # queue, it createaswrapper arround job
+      # Active Job has different structure how it is loading data from Sidekiq
+      # queue, it creates a wrapper around job.
       def active_job_message
         {
           'class'        => 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper',
+          'wrapped'      => @klass,
           'queue'        => @queue_name_with_prefix,
           'description'  => @description,
           'args'         => [{
             'job_class'  => @klass,
             'job_id'     => SecureRandom.uuid,
             'queue_name' => @queue_name_with_prefix,
-            'arguments'  => @args
+            'arguments'  => enqueue_args
           }]
         }
       end
 
-      # load cron jobs from Hash
-      # input structure should look like:
+      # Load cron jobs from Hash.
+      # Input structure should look like:
       # {
       #   'name_of_job' => {
       #     'class'       => 'MyClass',
@@ -150,22 +163,22 @@ module Sidekiq
       # }
       #
       def self.load_from_hash hash
-        array = hash.inject([]) do |out,(key, job)|
+        array = hash.map do |key, job|
           job['name'] = key
-          out << job
+          job
         end
         load_from_array array
       end
 
-      # like to {#load_from_hash}
-      # If exists old jobs in redis but removed from args, destroy old jobs
+      # Like #load_from_hash.
+      # If exists old jobs in Redis but removed from args, destroy old jobs.
       def self.load_from_hash! hash
         destroy_removed_jobs(hash.keys)
         load_from_hash(hash)
       end
 
-      # load cron jobs from Array
-      # input structure should look like:
+      # Load cron jobs from Array.
+      # Input structure should look like:
       # [
       #   {
       #     'name'        => 'name_of_job',
@@ -190,27 +203,27 @@ module Sidekiq
         errors
       end
 
-      # like to {#load_from_array}
-      # If exists old jobs in redis but removed from args, destroy old jobs
+      # Like #load_from_array.
+      # If exists old jobs in Redis but removed from args, destroy old jobs.
       def self.load_from_array! array
         job_names = array.map { |job| job["name"] }
         destroy_removed_jobs(job_names)
         load_from_array(array)
       end
 
-      # get all cron jobs
+      # Get all cron jobs.
       def self.all
         job_hashes = nil
         Sidekiq.redis do |conn|
           set_members = conn.smembers(jobs_key)
-          job_hashes = conn.pipelined do
+          job_hashes = conn.pipelined do |pipeline|
             set_members.each do |key|
-              conn.hgetall(key)
+              pipeline.hgetall(key)
             end
           end
         end
         job_hashes.compact.reject(&:empty?).collect do |h|
-          # no need to fetch missing args from redis since we just got this hash from there
+          # No need to fetch missing args from Redis since we just got this hash from there
           Sidekiq::Cron::Job.new(h.merge(fetch_missing_args: false))
         end
       end
@@ -224,7 +237,7 @@ module Sidekiq
       end
 
       def self.find name
-        #if name is hash try to get name from it
+        # If name is hash try to get name from it.
         name = name[:name] || name['name'] if name.is_a?(Hash)
 
         output = nil
@@ -233,17 +246,17 @@ module Sidekiq
             output = Job.new conn.hgetall( redis_key(name) )
           end
         end
-        output
+        output if output && output.valid?
       end
 
-      # create new instance of cron job
+      # Create new instance of cron job.
       def self.create hash
         new(hash).save
       end
 
-      #destroy job by name
+      # Destroy job by name.
       def self.destroy name
-        #if name is hash try to get name from it
+        # If name is hash try to get name from it.
         name = name[:name] || name['name'] if name.is_a?(Hash)
 
         if job = find(name)
@@ -265,22 +278,25 @@ module Sidekiq
         @cron = args["cron"]
         @description = args["description"] if args["description"]
 
-        #get class from klass or class
+        # Get class from klass or class.
         @klass = args["klass"] || args["class"]
 
-        #set status of job
+        # Set status of job.
         @status = args['status'] || status_from_redis
 
-        #set last enqueue time - from args or from existing job
+        # Set last enqueue time - from args or from existing job.
         if args['last_enqueue_time'] && !args['last_enqueue_time'].empty?
-          @last_enqueue_time = Time.parse(args['last_enqueue_time'])
+          @last_enqueue_time = parse_enqueue_time(args['last_enqueue_time'])
         else
           @last_enqueue_time = last_enqueue_time_from_redis
         end
 
-        #get right arguments for job
+        # Get right arguments for job.
+        @symbolize_args = args["symbolize_args"] == true || ("#{args["symbolize_args"]}" =~ (/^(true|t|yes|y|1)$/i)) == 0 || false
         @args = args["args"].nil? ? [] : parse_args( args["args"] )
 
+        @date_as_argument = args["date_as_argument"] == true || ("#{args["date_as_argument"]}" =~ (/^(true|t|yes|y|1)$/i)) == 0 || false
+
         @active_job = args["active_job"] == true || ("#{args["active_job"]}" =~ (/^(true|t|yes|y|1)$/i)) == 0 || false
         @active_job_queue_name_prefix = args["queue_name_prefix"]
         @active_job_queue_name_delimiter = args["queue_name_delimiter"]
@@ -295,8 +311,8 @@ module Sidekiq
             "args"  => @args,
           }
 
-          #get right data for message
-          #only if message wasn't specified before
+          # Get right data for message,
+          # only if message wasn't specified before.
           klass_data = case @klass
             when Class
               @klass.get_sidekiq_options
@@ -304,21 +320,21 @@ module Sidekiq
               begin
                 Sidekiq::Cron::Support.constantize(@klass).get_sidekiq_options
               rescue Exception => e
-                #Unknown class
+                # Unknown class
                 {"queue"=>"default"}
               end
           end
 
           message_data = klass_data.merge(message_data)
-          #override queue if setted in config
-          #only if message is hash - can be string (dumped JSON)
+
+          # Override queue if setted in config,
+          # only if message is hash - can be string (dumped JSON).
           if args['queue']
             @queue = message_data['queue'] = args['queue']
           else
             @queue = message_data['queue'] || "default"
           end
 
-          #dump message as json
           @message = message_data
         end
 
@@ -347,6 +363,12 @@ module Sidekiq
         !enabled?
       end
 
+      def pretty_message
+        JSON.pretty_generate Sidekiq.load_json(message)
+      rescue JSON::ParserError
+        message
+      end
+
       def status_from_redis
         out = "enabled"
         if fetch_missing_args
@@ -362,27 +384,45 @@ module Sidekiq
         out = nil
         if fetch_missing_args
           Sidekiq.redis do |conn|
-            out = Time.parse(conn.hget(redis_key, "last_enqueue_time")) rescue nil
+            out = parse_enqueue_time(conn.hget(redis_key, "last_enqueue_time")) rescue nil
           end
         end
         out
       end
 
-      #export job data to hash
+      def jid_history_from_redis
+        out =
+          Sidekiq.redis do |conn|
+            conn.lrange(jid_history_key, 0, -1) rescue nil
+          end
+
+        out && out.map do |jid_history_raw|
+          Sidekiq.load_json jid_history_raw
+        end
+      end
+
+      # Export job data to hash.
       def to_hash
-        {
+        hash = {
           name: @name,
-          klass: @klass,
+          klass: @klass.to_s,
           cron: @cron,
           description: @description,
           args: @args.is_a?(String) ? @args : Sidekiq.dump_json(@args || []),
           message: @message.is_a?(String) ? @message : Sidekiq.dump_json(@message || {}),
           status: @status,
-          active_job: @active_job,
+          active_job: @active_job ? "1" : "0",
           queue_name_prefix: @active_job_queue_name_prefix,
           queue_name_delimiter: @active_job_queue_name_delimiter,
-          last_enqueue_time: @last_enqueue_time,
+          last_enqueue_time: serialized_last_enqueue_time,
+          symbolize_args: symbolize_args? ? "1" : "0",
         }
+
+        if date_as_argument?
+          hash.merge!(date_as_argument: "1")
+        end
+
+        hash
       end
 
       def errors
@@ -390,7 +430,7 @@ module Sidekiq
       end
 
       def valid?
-        #clear previos errors
+        # Clear previous errors.
         @errors = []
 
         errors << "'name' must be set" if @name.nil? || @name.size == 0
@@ -398,21 +438,15 @@ module Sidekiq
           errors << "'cron' must be set"
         else
           begin
-            cron = Rufus::Scheduler::CronLine.new(@cron)
-            cron.next_time(Time.now.utc).utc
-          rescue Exception => e
-            #fix for different versions of cron-parser
-            if e.message == "Bad Vixie-style specification bad"
-              errors << "'cron' -> #{@cron}: not a valid cronline"
-            else
-              errors << "'cron' -> #{@cron}: #{e.message}"
-            end
+            @parsed_cron = Fugit.do_parse_cronish(@cron)
+          rescue => e
+            errors << "'cron' -> #{@cron.inspect} -> #{e.class}: #{e.message}"
           end
         end
 
         errors << "'klass' (or class) must be set" unless klass_valid
 
-        !errors.any?
+        errors.empty?
       end
 
       def klass_valid
@@ -425,67 +459,75 @@ module Sidekiq
         end
       end
 
-      # add job to cron jobs
-      # input:
-      #   name: (string) - name of job
-      #   cron: (string: '* * * * *' - cron specification when to run job
-      #   class: (string|class) - which class to perform
-      # optional input:
-      #   queue: (string) - which queue to use for enquing (will override class queue)
-      #   args: (array|hash|nil) - arguments for permorm method
-
       def save
-        #if job is invalid return false
+        # If job is invalid, return false.
         return false unless valid?
 
         Sidekiq.redis do |conn|
 
-          #add to set of all jobs
-          conn.sadd self.class.jobs_key, redis_key
+          # Add to set of all jobs
+          conn.sadd self.class.jobs_key, [redis_key]
 
-          #add informations for this job!
+          # Add informations for this job!
           conn.hmset redis_key, *hash_to_redis(to_hash)
 
-          #add information about last time! - don't enque right after scheduler poller starts!
+          # Add information about last time! - don't enque right after scheduler poller starts!
           time = Time.now.utc
-          conn.zadd(job_enqueued_key, time.to_f.to_s, formated_last_time(time).to_s) unless conn.exists(job_enqueued_key)
+          exists = conn.public_send(REDIS_EXISTS_METHOD, job_enqueued_key)
+          conn.zadd(job_enqueued_key, time.to_f.to_s, formatted_last_time(time).to_s) unless exists == true || exists == 1
         end
-        logger.info { "Cron Jobs - add job with name: #{@name}" }
+        Sidekiq.logger.info { "Cron Jobs - added job with name: #{@name}" }
       end
 
       def save_last_enqueue_time
         Sidekiq.redis do |conn|
-          # update last enqueue time
-          conn.hset redis_key, 'last_enqueue_time', @last_enqueue_time
+          # Update last enqueue time.
+          conn.hset redis_key, 'last_enqueue_time', serialized_last_enqueue_time
+        end
+      end
+
+      def add_jid_history(jid)
+        jid_history = {
+          jid: jid,
+          enqueued: @last_enqueue_time
+        }
+
+        @history_size ||= (Sidekiq::Options[:cron_history_size] || 10).to_i - 1
+        Sidekiq.redis do |conn|
+          conn.lpush jid_history_key,
+                     Sidekiq.dump_json(jid_history)
+          # Keep only last 10 entries in a fifo manner.
+          conn.ltrim jid_history_key, 0, @history_size
         end
       end
 
-      # remove job from cron jobs by name
-      # input:
-      #   first arg: name (string) - name of job (must be same - case sensitive)
       def destroy
         Sidekiq.redis do |conn|
-          #delete from set
-          conn.srem self.class.jobs_key, redis_key
+          # Delete from set.
+          conn.srem self.class.jobs_key, [redis_key]
 
-          #delete runned timestamps
+          # Delete runned timestamps.
           conn.del job_enqueued_key
 
-          #delete main job
+          # Delete jid_history.
+          conn.del jid_history_key
+
+          # Delete main job.
           conn.del redis_key
         end
-        logger.info { "Cron Jobs - deleted job with name: #{@name}" }
+
+        Sidekiq.logger.info { "Cron Jobs - deleted job with name: #{@name}" }
       end
 
-      # remove all job from cron
+      # Remove all job from cron.
       def self.destroy_all!
         all.each do |job|
           job.destroy
         end
-        logger.info { "Cron Jobs - deleted all jobs" }
+        Sidekiq.logger.info { "Cron Jobs - deleted all jobs" }
       end
 
-      # remove "removed jobs" between current jobs and new jobs
+      # Remove "removed jobs" between current jobs and new jobs
       def self.destroy_removed_jobs new_job_names
         current_job_names = Sidekiq::Cron::Job.all.map(&:name)
         removed_job_names = current_job_names - new_job_names
@@ -496,23 +538,22 @@ module Sidekiq
       # Parse cron specification '* * * * *' and returns
       # time when last run should be performed
       def last_time now = Time.now.utc
-        Rufus::Scheduler::CronLine.new(@cron).previous_time(now.utc).utc
+        parsed_cron.previous_time(now.utc).utc
       end
 
-      def formated_enqueue_time now = Time.now.utc
+      def formatted_enqueue_time now = Time.now.utc
         last_time(now).getutc.to_f.to_s
       end
 
-      def formated_last_time now = Time.now.utc
+      def formatted_last_time now = Time.now.utc
         last_time(now).getutc.iso8601
       end
 
       def self.exists? name
-        out = false
-        Sidekiq.redis do |conn|
-          out = conn.exists redis_key name
+        out = Sidekiq.redis do |conn|
+          conn.public_send(REDIS_EXISTS_METHOD, redis_key(name))
         end
-        out
+        out == true || out == 1
       end
 
       def exists?
@@ -525,70 +566,158 @@ module Sidekiq
 
       private
 
+      def parsed_cron
+        @parsed_cron ||= Fugit.parse_cronish(@cron)
+      end
+
       def not_enqueued_after?(time)
         @last_enqueue_time.nil? || @last_enqueue_time.to_i < last_time(time).to_i
       end
 
       # Try parsing inbound args into an array.
-      # args from Redis will be encoded JSON;
-      # try to load JSON, then failover
-      # to string array.
+      # Args from Redis will be encoded JSON,
+      # try to load JSON, then failover to string array.
       def parse_args(args)
         case args
+        when GlobalID::Identification
+          [convert_to_global_id_hash(args)]
         when String
           begin
-            Sidekiq.load_json(args)
+            parsed_args = Sidekiq.load_json(args)
+            symbolize_args? ? symbolize_args(parsed_args) : parsed_args
           rescue JSON::ParserError
-            [*args]   # cast to string array
+            [*args]
           end
         when Hash
-          [args]      # just put hash into array
+          args = serialize_argument(args)
+          symbolize_args? ? [symbolize_args(args)] : [args]
         when Array
-          args        # do nothing, already array
+          args = serialize_argument(args)
+          symbolize_args? ? symbolize_args(args) : args
+        else
+          [*args]
+        end
+      end
+
+      def symbolize_args?
+        @symbolize_args
+      end
+
+      def symbolize_args(input)
+        if input.is_a?(Array)
+          input.map do |arg|
+            if arg.respond_to?(:symbolize_keys)
+              arg.symbolize_keys
+            else
+              arg
+            end
+          end
+        elsif input.is_a?(Hash) && input.respond_to?(:symbolize_keys)
+          input.symbolize_keys
         else
-          [*args]     # cast to string array
+          input
         end
       end
 
+      def parse_enqueue_time(timestamp)
+        DateTime.strptime(timestamp, LAST_ENQUEUE_TIME_FORMAT).to_time.utc
+      rescue ArgumentError
+        DateTime.parse(timestamp).to_time.utc
+      end
+
       def not_past_scheduled_time?(current_time)
-        last_cron_time = Rufus::Scheduler::CronLine.new(@cron).previous_time(current_time).utc
+        last_cron_time = parsed_cron.previous_time(current_time).utc
         return false if (current_time.to_i - last_cron_time.to_i) > 60
         true
       end
 
-      # Redis key for set of all cron jobs
+      # Redis key for set of all cron jobs.
       def self.jobs_key
         "cron_jobs"
       end
 
-      # Redis key for storing one cron job
+      # Redis key for storing one cron job.
       def self.redis_key name
         "cron_job:#{name}"
       end
 
-      # Redis key for storing one cron job
+      # Redis key for storing one cron job.
       def redis_key
         self.class.redis_key @name
       end
 
-      # Redis key for storing one cron job run times
-      # (when poller added job to queue)
+      # Redis key for storing one cron job run times (when poller added job to queue)
       def self.job_enqueued_key name
         "cron_job:#{name}:enqueued"
       end
 
-      # Redis key for storing one cron job run times
-      # (when poller added job to queue)
+      def self.jid_history_key name
+        "cron_job:#{name}:jid_history"
+      end
+
       def job_enqueued_key
         self.class.job_enqueued_key @name
       end
 
-      # Give Hash
-      # returns array for using it for redis.hmset
+      def jid_history_key
+        self.class.jid_history_key @name
+      end
+
+      # Give Hash returns array for using it for redis.hmset
       def hash_to_redis hash
-        hash.inject([]){ |arr,kv| arr + [kv[0], kv[1]] }
+        hash.flat_map{ |key, value| [key, value || ""] }
       end
 
+      def serialized_last_enqueue_time
+        @last_enqueue_time&.strftime(LAST_ENQUEUE_TIME_FORMAT)
+      end
+      
+      def convert_to_global_id_hash(argument)
+        { GLOBALID_KEY => argument.to_global_id.to_s }
+      rescue URI::GID::MissingModelIdError
+        raise "Unable to serialize #{argument.class} " \
+          "without an id. (Maybe you forgot to call save?)"
+      end
+
+      def deserialize_argument(argument)
+        case argument
+        when String
+          argument
+        when Array
+          argument.map { |arg| deserialize_argument(arg) }
+        when Hash
+          if serialized_global_id?(argument)
+            deserialize_global_id argument
+          else
+            argument.transform_values { |v| deserialize_argument(v) }
+          end
+        else
+          argument
+        end
+      end
+
+      def serialized_global_id?(hash)
+        hash.size == 1 && hash.include?(GLOBALID_KEY)
+      end
+
+      def deserialize_global_id(hash)
+        GlobalID::Locator.locate hash[GLOBALID_KEY]
+      end
+
+      def serialize_argument(argument)
+        case argument
+        when GlobalID::Identification
+          convert_to_global_id_hash(argument)
+        when Array
+          argument.map { |arg| serialize_argument(arg) }
+        when Hash
+          argument.each_with_object({}) do |(key, value), hash|
+            hash[key] = serialize_argument(value)
+          end
+        else
+          argument
+        end
+      end
     end
   end
 end