resque-scheduler 2.3.1 → 2.4.0

data/bin/resque-scheduler

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# vim:fileencoding=utf-8
+
+require 'resque_scheduler'
+ResqueScheduler::Cli.run!

data/examples/Rakefile

@@ -0,0 +1,2 @@
+# vim:fileencoding=utf-8
+require 'resque_scheduler/tasks'

data/examples/config/initializers/resque-web.rb

@@ -0,0 +1,37 @@
+# vim:fileencoding=utf-8
+
+require 'json'
+require 'yaml'
+require 'resque'
+
+redis_env_var = ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+Resque.redis = ENV[redis_env_var] || 'localhost:6379'
+
+require 'resque_scheduler'
+require 'resque_scheduler/server'
+
+schedule_yml = ENV['RESQUE_SCHEDULE_YML']
+if schedule_yml
+  if File.exist?(schedule_yml)
+    Resque.schedule = YAML.load_file(schedule_yml)
+  else
+    Resque.schedule = YAML.load(schedule_yml)
+  end
+end
+
+schedule_json = ENV['RESQUE_SCHEDULE_JSON']
+if schedule_json
+  if File.exist?(schedule_json)
+    Resque.schedule = JSON.parse(File.read(schedule_json))
+  else
+    Resque.schedule = JSON.parse(schedule_json)
+  end
+end
+
+class Putter
+  @queue = 'putting'
+
+  def self.perform(*args)
+    args.each { |arg| puts arg }
+  end
+end

data/examples/dynamic-scheduling/README.md

@@ -0,0 +1,28 @@
+Dynamic Scheduling Example
+==========================
+
+Possible workaround for
+https://github.com/resque/resque-scheduler/issues/269
+
+This folder contains just the relevant files you would have to put into
+a rails application.
+
+The problem we want to fix is that when resque-scheduler is restarted,
+any dynamically added jobs are wiped. To fix it, we will run a
+statically scheduled job that dynamically reschedules any missing
+dynamic schedules.
+
+This workaround uses both a dynamic schedule (every time a user is
+created, a schedule is dynamically added to send him a daily email) and
+a static schedule (a job runs hourly, starting 10 seconds after starting
+resque-scheduler, to check that there is a scheduled job to send an
+email for every user; missing schedules are added).
+
+This way even though a resque-scheduler restart wipes all dynamic
+schedules, they are recreated by the `fix_schedules` job that runs in
+the static schedule.  Even if dynamic schedules were lost for any reason
+(data loss in redis clusters, whatever), they will be recreated hourly.
+
+This workaround requires that enough information is saved in the
+database to recreate all dynamic schedules. In this case we create one
+dynamically scheduled job for every user in the database.

data/examples/dynamic-scheduling/app/jobs/fix_schedules_job.rb

@@ -0,0 +1,54 @@
+# vim:fileencoding=utf-8
+#
+# Background job to fix the schedule for email sending. Any missing
+# schedule will be added to resque-schedule.
+#
+# Recent resque-scheduler versions wipe all dynamic schedules when
+# restarting. This means all dynamic schedules, which are added via the
+# API, are wiped on each application redeployment. A workaround for this
+# sometimes undesirable behavior is to make this job part of a static
+# schedule (see config/initializers/resque.rb and
+# config/static_schedule.yml).  This job will be scheduled to run every
+# hour even after restarting resque-schedule, and will add back the
+# dynamic schedules that were wiped on restart. It also serves as
+# safeguard against schedules getting lost for any reason.
+#
+# For more detail about this unfortunate behavior of resque-scheduler see:
+#
+#   https://github.com/resque/resque-scheduler/issues/269
+#
+# The perform method of this class will be invoked from a Resque worker.
+
+class FixSchedulesJob
+  @queue = :send_emails
+
+  # Fix email sending schedules. Any user which does not have scheduled
+  # sending of emails will be detected, and the missing scheduled job
+  # will be added to resque-schedule.
+  #
+  # This method is intended to be invoked from Resque, which means it is
+  # performed in the background.
+  def self.perform
+    users_unscheduled = []
+
+    User.all.each do |user|
+      # get schedule for the user
+      schedule = Resque.get_schedule("send_email_#{user.id}")
+      # if a user has no schedule, add it to the array
+      if schedule.nil?
+        users_unscheduled << user
+      end
+    end
+
+    if users_unscheduled.length > 0
+      users_unscheduled.each do |user|
+        name = "send_email_#{user.id}"
+        config = {}
+        config[:class] = 'SendEmailJob'
+        config[:args] = user.id
+        config[:every] = '1d'
+        Resque.set_schedule(name, config)
+      end
+    end
+  end
+end

data/examples/dynamic-scheduling/app/jobs/send_email_job.rb

@@ -0,0 +1,9 @@
+# vim:fileencoding=utf-8
+
+class SendEmailJob
+  @queue = :send_emails
+
+  def self.perform(user_id)
+    # ... do whatever you have to do to send an email to the user
+  end
+end

data/examples/dynamic-scheduling/app/models/user.rb

@@ -0,0 +1,16 @@
+# vim:fileencoding=utf-8
+
+class User < ActiveRecord::Base
+  after_create :schedule_send_email
+  
+  private
+  
+  def schedule_send_email
+    name = "send_email_#{id}"
+    config = {}
+    config[:class] = 'SendEmailJob'
+    config[:args] = id
+    config[:every] = '1d'
+    Resque.set_schedule(name, config)
+  end
+end

data/examples/dynamic-scheduling/config/resque.yml

@@ -0,0 +1,4 @@
+---
+development: localhost:6379
+test: localhost:6379:1
+production: localhost:6379 # or wherever your redis-server is in production

data/examples/dynamic-scheduling/config/static_schedule.yml

@@ -0,0 +1,7 @@
+---
+FixSchedulesJob:
+  description: 'Add any missing email sending schedules'
+  queue: send_emails
+  every:
+    - '1h'
+    - :first_in: '10s'

data/examples/dynamic-scheduling/lib/tasks/resque.rake

@@ -0,0 +1,48 @@
+# vim:fileencoding=utf-8
+
+require 'resque/tasks'
+require 'resque_scheduler/tasks'
+require 'yaml'
+
+namespace :resque do
+  task :setup do
+    require 'resque'
+    require 'resque_scheduler'
+
+    rails_root = ENV['RAILS_ROOT'] || File.expand_path('../../../', __FILE__)
+    rails_env = ENV['RAILS_ENV'] || 'development'
+
+    # In resque-only servers we must require each job class individually,
+    # because we're not running the full Rails app
+    require "#{rails_root}/app/jobs/send_email_job"
+    require "#{rails_root}/app/jobs/fix_schedules_job"
+
+    resque_config = YAML.load_file(
+      File.join(rails_root.to_s, 'config', 'resque.yml')
+    )
+    Resque.redis = resque_config[rails_env]
+
+    # If you want to be able to dynamically change the schedule,
+    # uncomment this line.  A dynamic schedule can be updated via the
+    # Resque::Scheduler.set_schedule (and remove_schedule) methods.
+    # When dynamic is set to true, the scheduler process looks for
+    # schedule changes and applies them on the fly.
+    # Note: This feature is only available in >=2.0.0.
+    Resque::Scheduler.dynamic = true
+
+    # Load static schedule (only in background servers).
+    # The schedule doesn't need to be stored in a YAML, it just needs to
+    # be a hash.  YAML is usually the easiest.
+    Resque.schedule = YAML.load_file(
+      File.join(rails_root.to_s, 'config', 'static_schedule.yml')
+    )
+
+    Resque.before_fork do |job|
+      # Reconnect to the DB before running each job. Otherwise we get errors if
+      # the DB is restarted after starting Resque.
+      # Absolutely necessary on Heroku, otherwise we get a "PG::Error: SSL
+      # SYSCALL error: EOF detected" exception
+      ActiveRecord::Base.establish_connection
+    end
+  end
+end

data/examples/run-resque-web

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd "$(dirname $0)"
+exec resque-web --foreground --no-launch config/initializers/resque-web.rb

data/lib/resque-scheduler.rb

@@ -0,0 +1,2 @@
+# vim:fileencoding=utf-8
+require File.expand_path('../resque_scheduler', __FILE__)

data/lib/resque/scheduler.rb

@@ -3,44 +3,92 @@ require 'resque/scheduler_locking'
 require 'resque_scheduler/logger_builder'
 
 module Resque
-
   class Scheduler
-
     extend Resque::SchedulerLocking
 
     class << self
+      # Allows for block-style configuration
+      def configure
+        yield self
+      end
+
+      # Used in `#load_schedule_job`
+      attr_writer :env
+
+      def env
+        return @env if @env
+        @env ||= Rails.env if defined?(Rails)
+        @env ||= ENV['RAILS_ENV']
+        @env
+      end
 
       # If true, logs more stuff...
-      attr_accessor :verbose
+      attr_writer :verbose
+
+      def verbose
+        @verbose ||= !!ENV['VERBOSE']
+      end
 
       # If set, produces no output
-      attr_accessor :mute
+      attr_writer :mute
+
+      def mute
+        @mute ||= !!ENV['MUTE']
+      end
 
       # If set, will write messages to the file
-      attr_accessor :logfile
+      attr_writer :logfile
+
+      def logfile
+        @logfile ||= ENV['LOGFILE']
+      end
+
+      # Sets whether to log in 'text' or 'json'
+      attr_writer :logformat
+
+      def logformat
+        @logformat ||= ENV['LOGFORMAT']
+      end
 
       # If set, will try to update the schedule in the loop
-      attr_accessor :dynamic
+      attr_writer :dynamic
+
+      def dynamic
+        @dynamic ||= !!ENV['DYNAMIC_SCHEDULE']
+      end
+
+      # If set, will append the app name to procline
+      attr_writer :app_name
+
+      def app_name
+        @app_name ||= ENV['APP_NAME']
+      end
 
       # Amount of time in seconds to sleep between polls of the delayed
       # queue.  Defaults to 5
       attr_writer :poll_sleep_amount
 
+      def poll_sleep_amount
+        @poll_sleep_amount ||=
+          Float(ENV.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
+      end
+
       attr_writer :logger
 
+      def logger
+        @logger ||= ResqueScheduler::LoggerBuilder.new(
+          :mute => mute,
+          :verbose => verbose,
+          :log_dev => logfile,
+          :format => logformat
+        ).build
+      end
+
       # the Rufus::Scheduler jobs that are scheduled
       def scheduled_jobs
         @@scheduled_jobs
       end
 
-      def poll_sleep_amount
-        @poll_sleep_amount ||= 5 # seconds
-      end
-
-      def logger
-        @logger ||= ResqueScheduler::LoggerBuilder.new(:mute => mute, :verbose => verbose, :log_dev => logfile).build
-      end
-
       # Schedule all jobs and continually look for delayed jobs (never returns)
       def run
         $0 = "resque-scheduler: Starting"
@@ -62,6 +110,8 @@ module Resque
           load_schedule!
         end
 
+        @th = Thread.current
+
         # Now start the scheduling part of the loop.
         loop do
           if is_master?
@@ -78,7 +128,6 @@ module Resque
         # never gets here.
       end
 
-
       # For all signals, set the shutdown flag and wait for current
       # poll/enqueing to finish (should be almost istant).  In the
       # case of sleeping, exit immediately.
@@ -139,13 +188,17 @@ module Resque
         args
       end
 
-      # Loads a job schedule into the Rufus::Scheduler and stores it in @@scheduled_jobs
+      # Loads a job schedule into the Rufus::Scheduler and stores it in
+      # @@scheduled_jobs
       def load_schedule_job(name, config)
-        # If rails_env is set in the config, enforce ENV['RAILS_ENV'] as
-        # required for the jobs to be scheduled.  If rails_env is missing, the
-        # job should be scheduled regardless of what ENV['RAILS_ENV'] is set
-        # to.
-        if config['rails_env'].nil? || rails_env_matches?(config)
+        # If `rails_env` or `env` is set in the config, load jobs only if they
+        # are meant to be loaded in `Resque::Scheduler.env`.  If `rails_env` or
+        # `env` is missing, the job should be scheduled regardless of the value
+        # of `Resque::Scheduler.env`.
+
+        configured_env = config['rails_env'] || config['env']
+
+        if configured_env.nil? || env_matches?(configured_env)
           log! "Scheduling #{name} "
           interval_defined = false
           interval_types = %w{cron every}
@@ -165,13 +218,25 @@ module Resque
           unless interval_defined
             log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
           end
+        else
+          log "Skipping schedule of #{name} because configured " <<
+              "env #{configured_env.inspect} does not match current " <<
+              "env #{env.inspect}"
         end
       end
 
-      # Returns true if the given schedule config hash matches the current
-      # ENV['RAILS_ENV']
+      # Returns true if the given schedule config hash matches the current env
       def rails_env_matches?(config)
-        config['rails_env'] && ENV['RAILS_ENV'] && config['rails_env'].gsub(/\s/,'').split(',').include?(ENV['RAILS_ENV'])
+        warn '`Resque::Scheduler.rails_env_matches?` is deprecated. ' <<
+             'Please use `Resque::Scheduler.env_matches?` instead.'
+        config['rails_env'] && env &&
+          config['rails_env'].split(/[\s,]+/).include?(env)
+      end
+
+      # Returns true if the current env is non-nil and the configured env
+      # (which is a comma-split string) includes the current env.
+      def env_matches?(configured_env)
+        env && configured_env.split(/[\s,]+/).include?(env)
       end
 
       # Handles queueing delayed items
@@ -302,41 +367,65 @@ module Resque
 
       # Sleeps and returns true
       def poll_sleep
-        @sleeping = true
-        handle_shutdown { sleep poll_sleep_amount }
-        @sleeping = false
+        handle_shutdown do
+          begin
+            begin
+              @sleeping = true
+              sleep poll_sleep_amount
+              @sleeping = false
+            rescue Interrupt
+              if @shutdown
+                Resque.clean_schedules
+                release_master_lock!
+              end
+            end
+          ensure
+            @sleeping = false
+          end
+        end
         true
       end
 
       # Sets the shutdown flag, clean schedules and exits if sleeping
       def shutdown
+        return if @shutdown
         @shutdown = true
-
-        if @sleeping
-          thread = Thread.new do
-            Resque.clean_schedules
-            release_master_lock!
-          end
-          thread.join
-          exit
-        end
+        log!('Shutting down')
+        @th.raise Interrupt if @sleeping
       end
 
       def log!(msg)
-        logger.info msg
+        logger.info { msg }
       end
 
       def log(msg)
-        logger.debug msg
+        logger.debug { msg }
       end
 
       def procline(string)
         log! string
-        $0 = "resque-scheduler-#{ResqueScheduler::VERSION}: #{string}"
+        argv0 = build_procline(string)
+        log "Setting procline #{argv0.inspect}"
+        $0 = argv0
       end
 
-    end
+      private
 
-  end
+      def app_str
+        app_name ? "[#{app_name}]" : ''
+      end
 
+      def env_str
+        env ? "[#{env}]" : ''
+      end
+
+      def build_procline(string)
+        "#{internal_name}#{app_str}#{env_str}: #{string}"
+      end
+
+      def internal_name
+        "resque-scheduler-#{ResqueScheduler::VERSION}"
+      end
+    end
+  end
 end