dynamodb-sidekiq-scheduler 0.0.1

data/lib/sidekiq-scheduler/scheduler.rb

@@ -0,0 +1,356 @@
+require 'rufus/scheduler'
+require 'json'
+require 'sidekiq-scheduler/rufus_utils'
+require 'sidekiq-scheduler/redis_manager'
+require 'sidekiq-scheduler/config'
+
+module SidekiqScheduler
+  class Scheduler
+    # We expect rufus jobs to have #params
+    Rufus::Scheduler::Job.module_eval do
+      alias_method :params, :opts
+    end
+
+    # TODO: Can we remove those attr_accessor's? If we need to keep them, we should
+    # update those values on the config object instead of just here in the scheduler.
+    # That's why we need to do what we do in the set_current_scheduler_options (not
+    # saying we will have to do it somehow still)
+    #
+    # NOTE: ^ Keeping this TODO here for now, in a future version of this project
+    # we will remove those attr acessors and use only our config object. For now,
+    # let's keep as it is.
+
+    # Set to enable or disable the scheduler.
+    attr_accessor :enabled
+
+    # Set to update the schedule in runtime in a given time period.
+    attr_accessor :dynamic
+
+    # Set to update the schedule in runtime dynamically per this period.
+    attr_accessor :dynamic_every
+
+    # Set to schedule jobs only when will be pushed to queues listened by sidekiq
+    attr_accessor :listened_queues_only
+
+    # Set custom options for rufus scheduler, like max_work_threads.
+    attr_accessor :rufus_scheduler_options
+
+    class << self
+
+      def instance
+        @instance = new unless @instance
+        @instance
+      end
+
+      def instance=(value)
+        @instance = value
+      end
+
+      def method_missing(method, *arguments, &block)
+        instance_methods.include?(method) ? instance.public_send(method, *arguments) : super
+      end
+    end
+
+    def initialize(config = SidekiqScheduler::Config.new(without_defaults: true))
+      @scheduler_config = config
+
+      self.enabled = config.enabled?
+      self.dynamic = config.dynamic?
+      self.dynamic_every = config.dynamic_every?
+      self.listened_queues_only = config.listened_queues_only?
+      self.rufus_scheduler_options = config.rufus_scheduler_options
+    end
+
+    # the Rufus::Scheduler jobs that are scheduled
+    def scheduled_jobs
+      @scheduled_jobs
+    end
+
+    def print_schedule
+      if rufus_scheduler
+        Sidekiq.logger.info "Scheduling Info\tLast Run"
+        scheduler_jobs = rufus_scheduler.jobs
+        scheduler_jobs.each_value do |v|
+          Sidekiq.logger.info "#{v.t}\t#{v.last}\t"
+        end
+      end
+    end
+
+    # Pulls the schedule from Sidekiq.schedule and loads it into the
+    # rufus scheduler instance
+    def load_schedule!
+      if enabled
+        Sidekiq.logger.info 'Loading Schedule'
+
+        # Load schedule from redis for the first time if dynamic
+        if dynamic
+          Sidekiq.reload_schedule!
+          @current_changed_score = Time.now.to_f
+          rufus_scheduler.every(dynamic_every) do
+            update_schedule
+          end
+        end
+
+        Sidekiq.logger.info 'Schedule empty! Set Sidekiq.schedule' if Sidekiq.schedule.empty?
+
+        @scheduled_jobs = {}
+        queues = scheduler_config.sidekiq_queues
+
+        Sidekiq.schedule.each do |name, config|
+          if !listened_queues_only || enabled_queue?(config['queue'].to_s, queues)
+            load_schedule_job(name, config)
+          else
+            Sidekiq.logger.info { "Ignoring #{name}, job's queue is not enabled." }
+          end
+        end
+
+        Sidekiq.logger.info 'Schedules Loaded'
+      else
+        Sidekiq.logger.info 'SidekiqScheduler is disabled'
+      end
+    end
+
+    # Loads a job schedule into the Rufus::Scheduler and stores it in @scheduled_jobs
+    def load_schedule_job(name, config)
+      # If rails_env is set in the config, enforce ENV['RAILS_ENV'] as
+      # required for the jobs to be scheduled.  If rails_env is missing, the
+      # job should be scheduled regardless of what ENV['RAILS_ENV'] is set
+      # to.
+      if config['rails_env'].nil? || rails_env_matches?(config)
+        Sidekiq.logger.info "Scheduling #{name} #{config}"
+        interval_defined = false
+        interval_types = %w(cron every at in interval)
+        interval_types.each do |interval_type|
+          config_interval_type = config[interval_type]
+
+          if !config_interval_type.nil? && config_interval_type.length > 0
+
+            schedule, options = SidekiqScheduler::RufusUtils.normalize_schedule_options(config_interval_type)
+
+            rufus_job = new_job(name, interval_type, config, schedule, options)
+            @scheduled_jobs[name] = rufus_job
+            SidekiqScheduler::Utils.update_job_next_time(name, rufus_job.next_time)
+
+            interval_defined = true
+
+            break
+          end
+        end
+
+        unless interval_defined
+          Sidekiq.logger.info "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
+        end
+      end
+    end
+
+    # Pushes the job into Sidekiq if not already pushed for the given time
+    #
+    # @param [String] job_name The job's name
+    # @param [Time] time The time when the job got cleared for triggering
+    # @param [Hash] config Job's config hash
+    def idempotent_job_enqueue(job_name, time, config)
+      registered = SidekiqScheduler::RedisManager.register_job_instance(job_name, time)
+
+      if registered
+        Sidekiq.logger.info "queueing #{config['class']} (#{job_name})"
+
+        handle_errors { enqueue_job(config, time) }
+
+        SidekiqScheduler::RedisManager.remove_elder_job_instances(job_name)
+      else
+        Sidekiq.logger.debug { "Ignoring #{job_name} job as it has been already enqueued" }
+      end
+    end
+
+    # Enqueue a job based on a config hash
+    #
+    # @param job_config [Hash] the job configuration
+    # @param time [Time] time the job is enqueued
+    def enqueue_job(job_config, time = Time.now)
+      config = prepare_arguments(job_config.dup)
+
+      if config.delete('include_metadata')
+        config['args'] = arguments_with_metadata(config['args'], "scheduled_at" => time.to_f.round(3))
+      end
+
+      if SidekiqScheduler::Utils.active_job_enqueue?(config['class'])
+        SidekiqScheduler::Utils.enqueue_with_active_job(config)
+      else
+        SidekiqScheduler::Utils.enqueue_with_sidekiq(config)
+      end
+    end
+
+    def rufus_scheduler
+      @rufus_scheduler ||= SidekiqScheduler::Utils.new_rufus_scheduler(rufus_scheduler_options)
+    end
+
+    # Stops old rufus scheduler and creates a new one.  Returns the new
+    # rufus scheduler
+    #
+    # @param [Symbol] stop_option The option to be passed to Rufus::Scheduler#stop
+    def clear_schedule!(stop_option = :wait)
+      if @rufus_scheduler
+        @rufus_scheduler.stop(stop_option)
+        @rufus_scheduler = nil
+      end
+
+      @@scheduled_jobs = {}
+
+      rufus_scheduler
+    end
+
+    def reload_schedule!
+      if enabled
+        Sidekiq.logger.info 'Reloading Schedule'
+        clear_schedule!
+        load_schedule!
+      else
+        Sidekiq.logger.info 'SidekiqScheduler is disabled'
+      end
+    end
+
+    def update_schedule
+      last_changed_score, @current_changed_score = @current_changed_score, Time.now.to_f
+      schedule_changes = SidekiqScheduler::RedisManager.get_schedule_changes(last_changed_score, @current_changed_score)
+
+      if schedule_changes.size > 0
+        Sidekiq.logger.info 'Updating schedule'
+
+        Sidekiq.reload_schedule!
+        schedule_changes.each do |schedule_name|
+          if Sidekiq.schedule.keys.include?(schedule_name)
+            unschedule_job(schedule_name)
+            load_schedule_job(schedule_name, Sidekiq.schedule[schedule_name])
+          else
+            unschedule_job(schedule_name)
+          end
+        end
+        Sidekiq.logger.info 'Schedule updated'
+      end
+    end
+
+    def job_enabled?(name)
+      job = Sidekiq.schedule[name]
+      schedule_state(name).fetch('enabled', job.fetch('enabled', true)) if job
+    end
+
+    def toggle_job_enabled(name)
+      state = schedule_state(name)
+      state['enabled'] = !job_enabled?(name)
+      set_schedule_state(name, state)
+    end
+
+    def toggle_all_jobs(new_state)
+      Sidekiq.schedule!.keys.each do |name|
+        state = schedule_state(name)
+        state['enabled'] = new_state
+        set_schedule_state(name, state)
+      end
+    end
+
+    private
+
+    attr_reader :scheduler_config
+
+    def new_job(name, interval_type, config, schedule, options)
+      options = options.merge({ :job => true, :tags => [name] })
+
+      rufus_scheduler.send(interval_type, schedule, options) do |job, time|
+        idempotent_job_enqueue(name, time, SidekiqScheduler::Utils.sanitize_job_config(config)) if job_enabled?(name)
+      end
+    end
+
+    def unschedule_job(name)
+      if scheduled_jobs[name]
+        Sidekiq.logger.debug "Removing schedule #{name}"
+        scheduled_jobs[name].unschedule
+        scheduled_jobs.delete(name)
+      end
+    end
+
+    # Retrieves a schedule state
+    #
+    # @param name [String] with the schedule's name
+    # @return [Hash] with the schedule's state
+    def schedule_state(name)
+      state = SidekiqScheduler::RedisManager.get_job_state(name)
+
+      state ? JSON.parse(state) : {}
+    end
+
+    # Saves a schedule state
+    #
+    # @param name [String] with the schedule's name
+    # @param state [Hash] with the schedule's state
+    def set_schedule_state(name, state)
+      SidekiqScheduler::RedisManager.set_job_state(name, state)
+    end
+
+    # Adds a Hash with schedule metadata as the last argument to call the worker.
+    # It currently returns the schedule time as a Float number representing the milisencods
+    # since epoch.
+    #
+    # @example with hash argument
+    #   arguments_with_metadata({value: 1}, scheduled_at: Time.now.round(3))
+    #   #=> [{value: 1}, {scheduled_at: <miliseconds since epoch>}]
+    #
+    # @param args [Array|Hash]
+    # @param metadata [Hash]
+    # @return [Array] arguments with added metadata
+    def arguments_with_metadata(args, metadata)
+      if args.is_a? Array
+        [*args, metadata]
+      else
+        [args, metadata]
+      end
+    end
+
+    # Returns true if a job's queue is included in the array of queues
+    #
+    # If queues are empty, returns true.
+    #
+    # @param [String] job_queue Job's queue name
+    # @param [Array<String>] queues
+    #
+    # @return [Boolean]
+    def enabled_queue?(job_queue, queues)
+      queues.empty? || queues.include?(job_queue)
+    end
+
+    # Convert the given arguments in the format expected to be enqueued.
+    #
+    # @param [Hash] config the options to be converted
+    # @option config [String] class the job class
+    # @option config [Hash/Array] args the arguments to be passed to the job
+    #   class
+    #
+    # @return [Hash]
+    def prepare_arguments(config)
+      config['class'] = SidekiqScheduler::Utils.try_to_constantize(config['class'])
+
+      if config['args'].is_a?(Hash)
+        config['args'].symbolize_keys! if config['args'].respond_to?(:symbolize_keys!)
+      else
+        config['args'] = Array(config['args'])
+      end
+
+      config
+    end
+
+    # Returns true if the given schedule config hash matches the current ENV['RAILS_ENV']
+    # @param [Hash] config The schedule job configuration
+    #
+    # @return [Boolean] true if the schedule config matches the current ENV['RAILS_ENV']
+    def rails_env_matches?(config)
+      config['rails_env'] && ENV['RAILS_ENV'] && config['rails_env'].gsub(/\s/, '').split(',').include?(ENV['RAILS_ENV'])
+    end
+
+    def handle_errors
+      begin
+        yield
+      rescue StandardError => e
+        Sidekiq.logger.info "#{e.class.name}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/sidekiq-scheduler/sidekiq_adapter.rb

@@ -0,0 +1,80 @@
+module SidekiqScheduler
+  class OptionNotSupportedAnymore < StandardError; end
+
+  class SidekiqAdapter
+    SIDEKIQ_GTE_6_5_0 = Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('6.5.0')
+    SIDEKIQ_GTE_7_0_0 = Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new('7.0.0')
+
+    def self.fetch_scheduler_config_from_sidekiq(sidekiq_config)
+      return {} if sidekiq_config.nil?
+
+      check_using_old_sidekiq_scheduler_config!(sidekiq_config)
+
+      if SIDEKIQ_GTE_6_5_0
+        sidekiq_config.fetch(:scheduler, {})
+      else
+        sidekiq_config.options.fetch(:scheduler, {})
+      end
+    end
+
+    def self.check_using_old_sidekiq_scheduler_config!(sidekiq_config)
+      %i[enabled dynamic dynamic_every schedule listened_queues_only rufus_scheduler_options].each do |option|
+        if SIDEKIQ_GTE_7_0_0
+          if sidekiq_config.key?(option)
+            raise OptionNotSupportedAnymore, ":#{option} option should be under the :scheduler: key"
+          end
+        elsif SIDEKIQ_GTE_6_5_0
+          unless sidekiq_config[option].nil?
+            raise OptionNotSupportedAnymore, ":#{option} option should be under the :scheduler: key"
+          end
+        else
+          if sidekiq_config.options.key?(option)
+            raise OptionNotSupportedAnymore, ":#{option} option should be under the :scheduler: key"
+          end
+        end
+      end
+    end
+
+    def self.start_schedule_manager(sidekiq_config:, schedule_manager:)
+      if SIDEKIQ_GTE_6_5_0
+        sidekiq_config[:schedule_manager] = schedule_manager
+        sidekiq_config[:schedule_manager].start
+      else
+        sidekiq_config.options[:schedule_manager] = schedule_manager
+        sidekiq_config.options[:schedule_manager].start
+      end
+    end
+
+    def self.stop_schedule_manager(sidekiq_config:)
+      if SIDEKIQ_GTE_6_5_0
+        sidekiq_config[:schedule_manager].stop
+      else
+        sidekiq_config.options[:schedule_manager].stop
+      end
+    end
+
+    def self.sidekiq_queues(sidekiq_config)
+      if SIDEKIQ_GTE_7_0_0
+        if sidekiq_config.nil? || (sidekiq_config.respond_to?(:empty?) && sidekiq_config.empty?)
+          Sidekiq.instance_variable_get(:@config).queues.map(&:to_s)
+        else
+          sidekiq_config.queues.map(&:to_s)
+        end
+      elsif SIDEKIQ_GTE_6_5_0
+        Sidekiq[:queues].map(&:to_s)
+      else
+        Sidekiq.options[:queues].map(&:to_s)
+      end
+    end
+
+    def self.redis_key_exists?(key_name)
+      Sidekiq.redis do |r|
+        if SIDEKIQ_GTE_7_0_0
+          r.exists(key_name) > 0
+        else
+          r.exists?(key_name)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq-scheduler/utils.rb

@@ -0,0 +1,143 @@
+require 'set'
+
+module SidekiqScheduler
+  module Utils
+
+    RUFUS_METADATA_KEYS = %w(description at cron every in interval enabled)
+
+    # Stringify keys belonging to a hash.
+    #
+    # Also stringifies nested keys and keys of hashes inside arrays, and sets
+    #
+    # @param [Object] object
+    #
+    # @return [Object]
+    def self.stringify_keys(object)
+      if object.is_a?(Hash)
+        Hash[[*object.map { |k, v| [k.to_s, stringify_keys(v) ]} ]]
+
+      elsif object.is_a?(Array) || object.is_a?(Set)
+        object.map { |v| stringify_keys(v) }
+
+      else
+        object
+      end
+    end
+
+    # Symbolize keys belonging to a hash.
+    #
+    # Also symbolizes nested keys and keys of hashes inside arrays, and sets
+    #
+    # @param [Object] object
+    #
+    # @return [Object]
+    def self.symbolize_keys(object)
+      if object.is_a?(Hash)
+        Hash[[*object.map { |k, v| [k.to_sym, symbolize_keys(v) ]} ]]
+
+      elsif object.is_a?(Array) || object.is_a?(Set)
+        object.map { |v| symbolize_keys(v) }
+
+      else
+        object
+      end
+    end
+
+    # Constantize a given string.
+    #
+    # @param [String] klass The string to constantize
+    #
+    # @return [Class] the class corresponding to the klass param
+    def self.try_to_constantize(klass)
+      klass.is_a?(String) ? Object.const_get(klass) : klass
+    rescue NameError
+      klass
+    end
+
+    # Initializes active_job using the passed parameters.
+    #
+    # @param [Class] klass The class to initialize
+    # @param [Array, Hash] args The parameters passed to the klass initializer
+    #
+    # @return [Object] instance of the class klass
+    def self.initialize_active_job(klass, args)
+      if args.is_a?(Array)
+        klass.new(*args)
+      else
+        klass.new(args)
+      end
+    end
+
+    # Returns true if the enqueuing needs to be done for an ActiveJob
+    #  class false otherwise.
+    #
+    # @param [Class] klass the class to check is decendant from ActiveJob
+    #
+    # @return [Boolean]
+    def self.active_job_enqueue?(klass)
+      klass.is_a?(Class) && defined?(ActiveJob::Enqueuing) &&
+        klass.included_modules.include?(ActiveJob::Enqueuing)
+    end
+
+    # Enqueues the job using the Sidekiq client.
+    #
+    # @param [Hash] config The job configuration
+    def self.enqueue_with_sidekiq(config)
+      Sidekiq::Client.push(sanitize_job_config(config))
+    end
+
+    # Enqueues the job using the ActiveJob.
+    #
+    # @param [Hash] config The job configuration
+    def self.enqueue_with_active_job(config)
+      options = {
+        queue: config['queue']
+      }.keep_if { |_, v| !v.nil? }
+
+      initialize_active_job(config['class'], config['args']).enqueue(options)
+    end
+
+    # Removes the hash values associated to the rufus metadata keys.
+    #
+    # @param [Hash] config The job configuration
+    #
+    # @return [Hash] the sanitized job config
+    def self.sanitize_job_config(config)
+      config.reject { |k, _| RUFUS_METADATA_KEYS.include?(k) }
+    end
+
+    # Creates a new instance of rufus scheduler.
+    #
+    # @return [Rufus::Scheduler] the scheduler instance
+    def self.new_rufus_scheduler(options = {})
+      Rufus::Scheduler.new(options).tap do |scheduler|
+        scheduler.define_singleton_method(:on_post_trigger) do |job, triggered_time|
+          if (job_name = job.tags[0])
+            SidekiqScheduler::Utils.update_job_last_time(job_name, triggered_time)
+            SidekiqScheduler::Utils.update_job_next_time(job_name, job.next_time)
+          end
+        end
+      end
+    end
+
+    # Pushes job's next time execution
+    #
+    # @param [String] name The job's name
+    # @param [Time] next_time The job's next time execution
+    def self.update_job_next_time(name, next_time)
+      if next_time
+        SidekiqScheduler::RedisManager.set_job_next_time(name, next_time)
+      else
+        SidekiqScheduler::RedisManager.remove_job_next_time(name)
+      end
+    end
+
+    # Pushes job's last execution time
+    #
+    # @param [String] name The job's name
+    # @param [Time] last_time The job's last execution time
+    def self.update_job_last_time(name, last_time)
+      SidekiqScheduler::RedisManager.set_job_last_time(name, last_time) if last_time
+    end
+  end
+end

data/lib/sidekiq-scheduler/version.rb

@@ -0,0 +1,3 @@
+module SidekiqScheduler
+  VERSION = '0.0.1'
+end

data/lib/sidekiq-scheduler/web.rb

@@ -0,0 +1,98 @@
+require 'sidekiq-scheduler'
+
+require_relative 'job_presenter'
+
+module SidekiqScheduler
+  # Hook into *Sidekiq::Web* app which adds a new '/recurring-jobs' page
+
+  module Web
+    VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
+
+    def self.registered(app)
+      app.get '/recurring-jobs' do
+        @presented_jobs = JobPresenter.build_collection(Sidekiq.schedule!)
+
+        erb File.read(File.join(VIEW_PATH, 'recurring_jobs.erb'))
+      end
+
+      app.post '/recurring-jobs/:name/enqueue' do
+        schedule = Sidekiq.get_schedule(params[:name])
+        SidekiqScheduler::Scheduler.instance.enqueue_job(schedule)
+        redirect "#{root_path}recurring-jobs"
+      end
+
+      app.post '/recurring-jobs/:name/toggle' do
+        Sidekiq.reload_schedule!
+
+        SidekiqScheduler::Scheduler.instance.toggle_job_enabled(params[:name])
+        redirect "#{root_path}recurring-jobs"
+      end
+
+      app.post '/recurring-jobs/:name/remove' do
+        Sidekiq.reload_schedule!
+
+        Sidekiq.remove_schedule(params[:name])
+        redirect "#{root_path}recurring-jobs"
+      end
+
+      app.post '/recurring-jobs/toggle-all' do
+        SidekiqScheduler::Scheduler.instance.toggle_all_jobs(params[:action] == 'enable')
+        redirect "#{root_path}recurring-jobs"
+      end
+
+      # New actions
+      app.post '/recurring-jobs/:name/destroy' do
+        Sidekiq.reload_schedule!
+
+        dynamo_clas_name = ENV.fetch('SIDEKIQ_SCHEDUER_DYNAMOID_CLASS', 'ScheduleRule')
+        klass = dynamo_clas_name.constantize
+        sr = klass.where(name: params[:name]).first
+        if sr.present?
+          sr.destroy
+        else
+          Sidekiq.remove_schedule(params[:name])
+        end
+
+        redirect "#{root_path}recurring-jobs"
+      end
+
+      app.post '/recurring-jobs/:name/edit' do
+        @title = 'Recurring Job Update'
+        @form_action = "#{root_path}recurring-jobs/#{ERB::Util.url_encode(params[:name])}/update"
+        @existing_rule = true
+        @name = params[:name]
+        @config = Sidekiq.get_schedule(params[:name])
+        erb File.read(File.join(VIEW_PATH, 'recurring_job.erb'))
+      end
+
+      app.post '/recurring-jobs/new' do
+        @title = 'New Recurring Job'
+        @existing_rule = false
+        @form_action = "#{root_path}recurring-jobs/create"
+        @name = ''
+        @config = { queue: 'default', enabled: false }
+        erb File.read(File.join(VIEW_PATH, 'recurring_job.erb'))
+      end
+
+      app.post '/recurring-jobs/:name/update' do
+        dynamo_clas_name = ENV.fetch('SIDEKIQ_SCHEDUER_DYNAMOID_CLASS', 'ScheduleRule')
+        klass = dynamo_clas_name.constantize
+        klass.update_or_create name: params[:name], config: JSON.parse(params[:config])
+
+        Sidekiq.reload_schedule!
+        redirect "#{root_path}recurring-jobs"
+      end
+
+      app.post '/recurring-jobs/create' do
+        dynamo_clas_name = ENV.fetch('SIDEKIQ_SCHEDUER_DYNAMOID_CLASS', 'ScheduleRule')
+        klass = dynamo_clas_name.constantize
+        klass.update_or_create name: params[:name], config: JSON.parse(params[:config])
+
+        Sidekiq.reload_schedule!
+        redirect "#{root_path}recurring-jobs"
+      end
+    end
+  end
+end
+
+require_relative 'extensions/web'