dynamodb-sidekiq-scheduler 0.0.1

data/lib/sidekiq-scheduler/config.rb

@@ -0,0 +1,80 @@
+module SidekiqScheduler
+  class Config
+    # We have to set the default as nil because the scheduler could be instantiated without
+    # passing the sidekiq config, and in those scenarios we don't want to fail
+    def initialize(sidekiq_config: nil, without_defaults: false)
+      @sidekiq_config = sidekiq_config
+      @scheduler_config = fetch_scheduler_config(sidekiq_config, without_defaults)
+    end
+
+    def enabled?
+      scheduler_config[:enabled]
+    end
+
+    def enabled=(value)
+      scheduler_config[:enabled] = value
+    end
+
+    def dynamic?
+      scheduler_config[:dynamic]
+    end
+
+    def dynamic=(value)
+      scheduler_config[:dynamic] = value
+    end
+
+    def dynamic_every?
+      scheduler_config[:dynamic_every]
+    end
+
+    def dynamic_every=(value)
+      scheduler_config[:dynamic_every] = value
+    end
+
+    def schedule
+      scheduler_config[:schedule]
+    end
+
+    def schedule=(value)
+      scheduler_config[:schedule] = value
+    end
+
+    def listened_queues_only?
+      scheduler_config[:listened_queues_only]
+    end
+
+    def listened_queues_only=(value)
+      scheduler_config[:listened_queues_only] = value
+    end
+
+    def rufus_scheduler_options
+      scheduler_config[:rufus_scheduler_options]
+    end
+
+    def rufus_scheduler_options=(value)
+      scheduler_config[:rufus_scheduler_options] = value
+    end
+
+    def sidekiq_queues
+      SidekiqScheduler::SidekiqAdapter.sidekiq_queues(sidekiq_config)
+    end
+
+    private
+
+    attr_reader :scheduler_config
+    attr_reader :sidekiq_config
+
+    DEFAULT_OPTIONS = {
+      enabled: true,
+      dynamic: false,
+      dynamic_every: '5s',
+      schedule: {},
+      rufus_scheduler_options: {}
+    }.freeze
+
+    def fetch_scheduler_config(sidekiq_config, without_defaults)
+      conf = SidekiqScheduler::SidekiqAdapter.fetch_scheduler_config_from_sidekiq(sidekiq_config)
+      without_defaults ? conf : DEFAULT_OPTIONS.merge(conf)
+    end
+  end
+end

data/lib/sidekiq-scheduler/extensions/schedule.rb

@@ -0,0 +1,4 @@
+require 'sidekiq'
+require_relative '../schedule'
+
+Sidekiq.extend SidekiqScheduler::Schedule

data/lib/sidekiq-scheduler/extensions/web.rb

@@ -0,0 +1,14 @@
+require 'sidekiq/web' unless defined?(Sidekiq::Web)
+
+ASSETS_PATH = File.expand_path('../../../web/assets', __dir__)
+
+Sidekiq::Web.register(SidekiqScheduler::Web)
+Sidekiq::Web.tabs['recurring_jobs'] = 'recurring-jobs'
+Sidekiq::Web.locales << File.expand_path("#{File.dirname(__FILE__)}/../../../web/locales")
+
+if Sidekiq::VERSION >= '6.0.0'
+  Sidekiq::Web.use Rack::Static, urls: ['/stylesheets-scheduler'],
+                                 root: ASSETS_PATH,
+                                 cascade: true,
+                                 header_rules: [[:all, { 'Cache-Control' => 'public, max-age=86400' }]]
+end

data/lib/sidekiq-scheduler/job_presenter.rb

@@ -0,0 +1,74 @@
+begin
+  require 'sidekiq/web/helpers'
+rescue LoadError
+  require 'sidekiq/web_helpers'
+end
+require 'sidekiq-scheduler/redis_manager'
+
+module SidekiqScheduler
+  class JobPresenter
+    attr_reader :name
+
+    include Sidekiq::WebHelpers
+
+    def initialize(name, attributes)
+      @name = name
+      @attributes = attributes
+    end
+
+    # Returns the next time execution for the job
+    #
+    # @return [String] with the job's next time
+    def next_time
+      execution_time = SidekiqScheduler::RedisManager.get_job_next_time(name)
+
+      relative_time(Time.parse(execution_time)) if execution_time
+    end
+
+    # Returns the last execution time for the job
+    #
+    # @return [String] with the job's last time
+    def last_time
+      execution_time = SidekiqScheduler::RedisManager.get_job_last_time(name)
+
+      relative_time(Time.parse(execution_time)) if execution_time
+    end
+
+    # Returns the interval for the job
+    #
+    # @return [String] with the job's interval
+    def interval
+      @attributes['cron'] || @attributes['interval'] || @attributes['every']
+    end
+
+    # Returns the queue of the job
+    #
+    # @return [String] with the job's queue
+    def queue
+      @attributes.fetch('queue', 'default')
+    end
+
+    # Delegates the :[] method to the attributes' hash
+    #
+    # @return [String] with the value for that key
+    def [](key)
+      @attributes[key]
+    end
+
+    def enabled?
+      SidekiqScheduler::Scheduler.job_enabled?(@name)
+    end
+
+    # Builds the presenter instances for the schedule hash
+    #
+    # @param schedule_hash [Hash] with the redis schedule
+    # @return [Array<JobPresenter>] an array with the instances of presenters
+    def self.build_collection(schedule_hash)
+      schedule_hash ||= {}
+
+      schedule_hash.sort.map do |name, job_spec|
+        new(name, job_spec)
+      end
+    end
+  end
+end

data/lib/sidekiq-scheduler/manager.rb

@@ -0,0 +1,44 @@
+require 'sidekiq-scheduler/schedule'
+require 'sidekiq-scheduler/scheduler'
+
+module SidekiqScheduler
+
+  # The delayed job router in the system.  This
+  # manages the scheduled jobs pushed messages
+  # from Redis onto the work queues
+  #
+  class Manager
+    def initialize(config)
+      set_current_scheduler_options(config)
+
+      @scheduler_instance = SidekiqScheduler::Scheduler.new(config)
+      SidekiqScheduler::Scheduler.instance = @scheduler_instance
+      Sidekiq.schedule = config.schedule if @scheduler_instance.enabled
+    end
+
+    def stop
+      @scheduler_instance.clear_schedule!
+    end
+
+    def start
+      @scheduler_instance.load_schedule!
+    end
+
+    private
+
+    def set_current_scheduler_options(config)
+      enabled = SidekiqScheduler::Scheduler.enabled
+      dynamic = SidekiqScheduler::Scheduler.dynamic
+      dynamic_every = SidekiqScheduler::Scheduler.dynamic_every
+      listened_queues_only = SidekiqScheduler::Scheduler.listened_queues_only
+
+      config.enabled = enabled unless enabled.nil?
+      config.dynamic = dynamic unless dynamic.nil?
+      config.dynamic_every = dynamic_every unless dynamic_every.nil?
+      unless Sidekiq.schedule.nil? || (Sidekiq.schedule.respond_to?(:empty?) && Sidekiq.schedule.empty?)
+        config.schedule = Sidekiq.schedule
+      end
+      config.listened_queues_only = listened_queues_only unless listened_queues_only.nil?
+    end
+  end
+end

data/lib/sidekiq-scheduler/redis_manager.rb

@@ -0,0 +1,241 @@
+module SidekiqScheduler
+  module RedisManager
+
+    REGISTERED_JOBS_THRESHOLD_IN_SECONDS = 24 * 60 * 60
+
+    # Returns the schedule of a given job
+    #
+    # @param [String] name The name of the job
+    #
+    # @return [String] schedule in JSON format
+    def self.get_job_schedule(name)
+      hget(schedules_key, name)
+    end
+
+    # Returns the state of a given job
+    #
+    # @param [String] name The name of the job
+    #
+    # @return [String] state in JSON format
+    def self.get_job_state(name)
+      hget(schedules_state_key, name)
+    end
+
+    # Returns the next execution time for a given job
+    #
+    # @param [String] name The name of the job
+    #
+    # @return [String] next time the job has to be executed
+    def self.get_job_next_time(name)
+      hget(next_times_key, name)
+    end
+
+    # Returns the last execution time of a given job
+    #
+    # @param [String] name The name of the job
+    #
+    # @return [String] last time the job was executed
+    def self.get_job_last_time(name)
+      hget(last_times_key, name)
+    end
+
+    # Sets the schedule for a given job
+    #
+    # @param [String] name The name of the job
+    # @param [Hash] config The new schedule for the job
+    def self.set_job_schedule(name, config)
+      hset(schedules_key, name, JSON.generate(config))
+    end
+
+    # Sets the state for a given job
+    #
+    # @param [String] name The name of the job
+    # @param [Hash] state The new state for the job
+    def self.set_job_state(name, state)
+      hset(schedules_state_key, name, JSON.generate(state))
+    end
+
+    # Sets the next execution time for a given job
+    #
+    # @param [String] name The name of the job
+    # @param [String] next_time The next time the job has to be executed
+    def self.set_job_next_time(name, next_time)
+      hset(next_times_key, name, String(next_time))
+    end
+
+    # Sets the last execution time for a given job
+    #
+    # @param [String] name The name of the job
+    # @param [String] last_time The last time the job was executed
+    def self.set_job_last_time(name, last_time)
+      hset(last_times_key, name, String(last_time))
+    end
+
+    # Removes the schedule for a given job
+    #
+    # @param [String] name The name of the job
+    def self.remove_job_schedule(name)
+      hdel(schedules_key, name)
+    end
+
+    # Removes the next execution time for a given job
+    #
+    # @param [String] name The name of the job
+    def self.remove_job_next_time(name)
+      hdel(next_times_key, name)
+    end
+
+    # Returns the schedules of all the jobs
+    #
+    # @return [Hash] hash with all the job schedules
+    def self.get_all_schedules
+      Sidekiq.redis { |r| r.hgetall(schedules_key) }
+    end
+
+    # Returns boolean value that indicates if the schedules value exists
+    #
+    # @return [Boolean] true if the schedules key is set, false otherwise
+    def self.schedule_exist?
+      SidekiqScheduler::SidekiqAdapter.redis_key_exists?(schedules_key)
+    end
+
+    # Returns all the schedule changes for a given time range.
+    #
+    # @param [Float] from The minimum value in the range
+    # @param [Float] to The maximum value in the range
+    #
+    # @return [Array] array with all the changed job names
+    def self.get_schedule_changes(from, to)
+      Sidekiq.redis { |r| r.zrangebyscore(schedules_changed_key, from, "(#{to}") }
+    end
+
+    # Register a schedule change for a given job
+    #
+    # @param [String] name The name of the job
+    def self.add_schedule_change(name)
+      Sidekiq.redis { |r| r.zadd(schedules_changed_key, Time.now.to_f, name) }
+    end
+
+    # Remove all the schedule changes records
+    def self.clean_schedules_changed
+      Sidekiq.redis { |r| r.del(schedules_changed_key) unless r.type(schedules_changed_key) == 'zset' }
+    end
+
+    # Removes a queued job instance
+    #
+    # @param [String] job_name The name of the job
+    # @param [Time] time The time at which the job was cleared by the scheduler
+    #
+    # @return [Boolean] true if the job was registered, false otherwise
+    def self.register_job_instance(job_name, time)
+      job_key = pushed_job_key(job_name)
+      registered, _ = Sidekiq.redis do |r|
+        r.pipelined do |pipeline|
+          pipeline.zadd(job_key, time.to_i, time.to_i)
+          pipeline.expire(job_key, REGISTERED_JOBS_THRESHOLD_IN_SECONDS)
+        end
+      end
+
+      registered.instance_of?(Integer) ? (registered > 0) : registered
+    end
+
+    # Removes instances of the job older than 24 hours
+    #
+    # @param [String] job_name The name of the job
+    def self.remove_elder_job_instances(job_name)
+      seconds_ago = Time.now.to_i - REGISTERED_JOBS_THRESHOLD_IN_SECONDS
+
+      Sidekiq.redis do |r|
+        r.zremrangebyscore(pushed_job_key(job_name), 0, seconds_ago)
+      end
+    end
+
+    # Returns the key of the Redis sorted set used to store job enqueues
+    #
+    # @param [String] job_name The name of the job
+    #
+    # @return [String] the pushed job key
+    def self.pushed_job_key(job_name)
+      "#{key_prefix}sidekiq-scheduler:pushed:#{job_name}"
+    end
+
+    # Returns the key of the Redis hash for job's execution times hash
+    #
+    # @return [String] with the key
+    def self.next_times_key
+      "#{key_prefix}sidekiq-scheduler:next_times"
+    end
+
+    # Returns the key of the Redis hash for job's last execution times hash
+    #
+    # @return [String] with the key
+    def self.last_times_key
+      "#{key_prefix}sidekiq-scheduler:last_times"
+    end
+
+    # Returns the Redis's key for saving schedule states.
+    #
+    # @return [String] with the key
+    def self.schedules_state_key
+      "#{key_prefix}sidekiq-scheduler:states"
+    end
+
+    # Returns the Redis's key for saving schedules.
+    #
+    # @return [String] with the key
+    def self.schedules_key
+      "#{key_prefix}schedules"
+    end
+
+    # Returns the Redis's key for saving schedule changes.
+    #
+    # @return [String] with the key
+    def self.schedules_changed_key
+      "#{key_prefix}schedules_changed"
+    end
+
+    # Returns the key prefix used to generate all scheduler keys
+    #
+    # @return [String] with the key prefix
+    def self.key_prefix
+      @key_prefix
+    end
+
+    # Sets the key prefix used to scope all scheduler keys
+    #
+    # @param [String] value The string to use as the prefix. A ":" will be appended as a delimiter if needed.
+    def self.key_prefix=(value)
+      value = "#{value}:" if value && !%w[. :].include?(value[-1])
+      @key_prefix = value
+    end
+
+    private
+
+    # Returns the value of a Redis stored hash field
+    #
+    # @param [String] hash_key The key name of the hash
+    # @param [String] field_key The key name of the field
+    #
+    # @return [String]
+    def self.hget(hash_key, field_key)
+      Sidekiq.redis { |r| r.hget(hash_key, field_key) }
+    end
+
+    # Sets the value of a Redis stored hash field
+    #
+    # @param [String] hash_key The key name of the hash
+    # @param [String] field_key The key name of the field
+    # @param [String] value The new value name for the field
+    def self.hset(hash_key, field_key, value)
+      Sidekiq.redis { |r| r.hset(hash_key, field_key, value) }
+    end
+
+    # Removes the value of a Redis stored hash field
+    #
+    # @param [String] hash_key The key name of the hash
+    # @param [String] field_key The key name of the field
+    def self.hdel(hash_key, field_key)
+      Sidekiq.redis { |r| r.hdel(hash_key, field_key) }
+    end
+  end
+end

data/lib/sidekiq-scheduler/rufus_utils.rb

@@ -0,0 +1,29 @@
+require 'sidekiq-scheduler/utils'
+
+module SidekiqScheduler
+  class RufusUtils
+
+    # Normalizes schedule options to rufus scheduler options
+    #
+    # @param options [String, Array]
+    #
+    # @return [Array]
+    #
+    # @example
+    #   normalize_schedule_options('15m') => ['15m', {}]
+    #   normalize_schedule_options(['15m']) => ['15m', {}]
+    #   normalize_schedule_options(['15m', first_in: '5m']) => ['15m', { first_in: '5m' }]
+    def self.normalize_schedule_options(options)
+      schedule, opts = options
+
+      if !opts.is_a?(Hash)
+        opts = {}
+      end
+
+      opts = SidekiqScheduler::Utils.symbolize_keys(opts)
+
+      return schedule, opts
+    end
+  end
+end
+

data/lib/sidekiq-scheduler/schedule.rb

@@ -0,0 +1,154 @@
+require 'json'
+
+require 'sidekiq-scheduler/utils'
+require_relative 'redis_manager'
+
+module SidekiqScheduler
+  module Schedule
+
+    # Accepts a new schedule configuration of the form:
+    #
+    #   {
+    #     "MakeTea" => {
+    #       "every" => "1m" },
+    #     "some_name" => {
+    #       "cron"        => "5/* * * *",
+    #       "class"       => "DoSomeWork",
+    #       "args"        => "work on this string",
+    #       "description" => "this thing works it"s butter off" },
+    #     ...
+    #   }
+    #
+    # Hash keys can be anything and are used to describe and reference
+    # the scheduled job. If the "class" argument is missing, the key
+    # is used implicitly as "class" argument - in the "MakeTea" example,
+    # "MakeTea" is used both as job name and sidekiq worker class.
+    #
+    # :cron can be any cron scheduling string
+    #
+    # :every can be used in lieu of :cron. see rufus-scheduler's 'every' usage
+    # for valid syntax. If :cron is present it will take precedence over :every.
+    #
+    # :class must be a sidekiq worker class. If it is missing, the job name (hash key)
+    # will be used as :class.
+    #
+    # :args can be any yaml which will be converted to a ruby literal and
+    # passed in a params. (optional)
+    #
+    # :description is just that, a description of the job (optional). If
+    # params is an array, each element in the array is passed as a separate
+    # param, otherwise params is passed in as the only parameter to perform.
+    def schedule=(schedule_hash)
+      schedule_hash = prepare_schedule(schedule_hash)
+      to_remove = get_all_schedules.keys - schedule_hash.keys.map(&:to_s)
+
+      schedule_hash.each do |name, job_spec|
+        set_schedule(name, job_spec)
+      end
+
+      to_remove.each do |name|
+        remove_schedule(name)
+      end
+
+      @schedule = schedule_hash
+    end
+
+    def schedule
+      @schedule
+    end
+
+    # Reloads the schedule from Redis and return it.
+    #
+    # @return Hash
+    def reload_schedule!
+      @schedule = get_schedule
+    end
+    alias_method :schedule!, :reload_schedule!
+
+    # Retrive the schedule configuration for the given name
+    # if the name is nil it returns a hash with all the
+    # names end their schedules.
+    def get_schedule(name = nil)
+      if name.nil?
+        get_all_schedules
+      else
+        encoded_schedule = SidekiqScheduler::RedisManager.get_job_schedule(name)
+        encoded_schedule.nil? ? nil : JSON.parse(encoded_schedule)
+      end
+    end
+
+    # gets the schedule as it exists in redis
+    def get_all_schedules
+      schedules = {}
+
+      if SidekiqScheduler::RedisManager.schedule_exist?
+        SidekiqScheduler::RedisManager.get_all_schedules.tap do |h|
+          h.each do |name, config|
+            schedules[name] = JSON.parse(config)
+          end
+        end
+      end
+
+      schedules
+    end
+
+    # Create or update a schedule with the provided name and configuration.
+    #
+    # Note: values for class and custom_job_class need to be strings,
+    # not constants.
+    #
+    #    Sidekiq.set_schedule('some_job', { :class => 'SomeJob',
+    #                                       :every => '15mins',
+    #                                       :queue => 'high',
+    #                                       :args => '/tmp/poop' })
+    def set_schedule(name, config)
+      existing_config = get_schedule(name)
+      unless existing_config && existing_config == config
+        SidekiqScheduler::RedisManager.set_job_schedule(name, config)
+        SidekiqScheduler::RedisManager.add_schedule_change(name)
+      end
+      config
+    end
+
+    # remove a given schedule by name
+    def remove_schedule(name)
+      SidekiqScheduler::RedisManager.remove_job_schedule(name)
+      SidekiqScheduler::RedisManager.add_schedule_change(name)
+    end
+
+    private
+
+    def prepare_schedule(schedule_hash)
+      schedule_hash = SidekiqScheduler::Utils.stringify_keys(schedule_hash)
+
+      prepared_hash = {}
+
+      schedule_hash.each do |name, job_spec|
+        job_spec = job_spec.dup
+
+        job_class = job_spec.fetch('class', name)
+        inferred_queue = infer_queue(job_class)
+
+        job_spec['class'] ||= job_class
+        job_spec['queue'] ||= inferred_queue unless inferred_queue.nil?
+
+        prepared_hash[name] = job_spec
+      end
+      prepared_hash
+    end
+
+    def infer_queue(klass)
+      klass = try_to_constantize(klass)
+
+      # ActiveJob uses queue_as when the job is created
+      # to determine the queue
+      if klass.respond_to?(:sidekiq_options) && !SidekiqScheduler::Utils.active_job_enqueue?(klass)
+        klass.sidekiq_options['queue']
+      end
+    end
+
+    def try_to_constantize(klass)
+      SidekiqScheduler::Utils.try_to_constantize(klass)
+    end
+  end
+end