rufus-scheduler 2.0.24 → 3.0.0

data/Rakefile

@@ -1,8 +1,5 @@
 
-$:.unshift('.') # 1.9.2
-
 require 'rubygems'
-require 'rubygems/user_interaction' if Gem::RubyGemsVersion == '1.5.0'
 
 require 'rake'
 require 'rake/clean'
@@ -42,7 +39,7 @@ desc %{
 task :build do
 
   sh "gem build #{GEMSPEC_FILE}"
-  sh "mkdir pkg" rescue nil
+  sh "mkdir -p pkg"
   sh "mv #{GEMSPEC.name}-#{GEMSPEC.version}.gem pkg/"
 end
 

data/TODO.txt

@@ -1,57 +1,147 @@
 
-[o] spec for jobs in the past (in and at)
-[o] :discard_past
-
-[o] every
-[o] cron
-
-[o] CHECK every and unschedule !!!
-
-[o] :tags
-[o] timeout feature (at/in/every/cron) in Job class
-
-[o] :first_in, :first_at
-
-[x] :dont_reschedule (or block returns false ?)
-
-[o] [get_]jobs methods
-[o] find methods
-
-[x] CTRL-C during tests : allow, trap_int...
-
-[o] 1.9
-[o] j1.2.0
-
-[o] revise trigger block arity
-    use a compatibility switch ? yes
-
-[o] synchronize @cron_jobs ?
-
-[o] why not : make it work even if EM is not present
-    EmScheduler < Scheduler
-    FiberScheduler < Scheduler
-
-[x] :blocking => 'blockname' idea, mutex = @mutexes['blockname'] ...
-[o] eventually, make sleep frequency customizable
-
-[o] PlainScheduler : name thread
-
-[o] document :blocking
-
-[o] README.rdoc
-[o] fix jruby120 --em
-
-[o] handle_exception (job, e)
-
-[o] Schedulable
-
-[o] Rufus::Scheduler.start_new() : autodetect EM ?
-[o] check :blocking and every (reschedule blocking...)
-[o] document :thread_name scheduler option
-
-[o] unify cron_jobs#trigger_matching_jobs(now) and jobs#job_to_trigger
-[o] pluggable job queues
-
-[ ] Joel's complaint about timeout jobs gone ballistic
-[x] move trigger_job out of the scheduler
+[o] merge schedule_queue and unschedule_queue (and merge [un]schedule steps)
+[x] OR stop using queue, since we've got the thread-safe JobArray
+[x] if possible, drop the mutex in JobArray
+    NO, that mutex is necessary for Scheduler#jobs (on JRuby an co)...
+[o] named mutexes
+[o] drop the schedule queue, rely on the mutex in JobArray
+[o] def jobs; (@jobs.to_a + running_jobs).uniq; end
+[o] replace @unscheduled by @unscheduled_at
+[o] make sure #jobs doesn't return unscheduled jobs
+[o] job tags and find_by_tag(t) (as in rs 2.x)
+[o] require tzinfo anyway (runtime dep)
+[o] document frequency
+[o] accept :frequency => '5s'
+[o] timeout (as in rufus-scheduler 2.x)
+[o] Rufus::Scheduler#running_jobs (as in rufus-scheduler 2.x)
+[o] Rufus::Scheduler#terminate_all_jobs
+[o] Rufus::Scheduler::Job#kill
+[x] Rufus::Scheduler#kill_all_jobs
+[o] Rufus::Scheduler#shutdown(:terminate or :kill (or nothing))
+[o] RepeatJob #pause / #resume (think about discard past)
+[o] Rufus::Scheduler.start_new (backward comp) (with deprec note?)
+[o] pass job to scheduled block? What does rs 2.x do?
+[o] :first[_in|_at] for RepeatJob
+[o] :last[_in|_at] for RepeatJob
+[o] :times for RepeatJob (how many recurrences)
+[o] fix issue #39 (first_at parses as UTC)
+[o] about issue #43, raise if cron/every job frequency < scheduler frequency
+[o] unlock spec/parse_spec.rb:30 "parse datimes with timezones"
+[o] some kind of Schedulable (Xyz#call(job, time))
+[o] add Jruby and Rubinius to Travis
+[o] make Job #first_at= / #last_at= automatically parse strings?
+[o] bring in Kratob's spec about mutex vs timeout and adapt 3.0 to it,
+    https://github.com/jmettraux/rufus-scheduler/pull/67
+[x] :unschedule_if => lambda { |job| ... }
+[o] OR look at how it was done in rs 2.0.x, some return value?
+    no, pass the job as arg to the block, then let the block do job.unschedule
+    so, document schedule.every('10d') { |j| j.unschedule if x?() }
+[x] remove the time in job.trigger(time)
+[o] add spec for job queued then unscheduled
+[o] add spec for Scheduler#shutdown and work threads
+[o] at some point, bring back rbx19 to Travis
+[o] move the parse/util part of scheduler.rb to util.rb
+[o] rescue KillSignal in job thread loop to kill just the job
+[o] add spec for raise if scheduling a job while scheduler is shutting down
+[o] schedule_in(2.days.from_now) {}
+    at and in could understand each others time parameter, ftw...
+    use the new #parse_to_time? no
+[o] do repeat jobs reschedule after timing out? yes
+[o] schedule_interval('20s')?
+[x] Scheduler#reschedule(job) (new copy of the job)
+[x] #free_all_work_threads is missing an implementation
+[x] rescue StandardError
+    :on_error => :crash[_scheduler]
+    :on_error => :ignore
+    :on_error => ...
+[o] on_error: what about TimeoutError in that scheme?
+    TimeoutError goes to $stderr, like a normal error
+[o] link to SO for support
+    - sublink to "how to report bugs effectively"
+[o] link to #ruote for support
+[x] lockblock? pass a block to teach the scheduler how to lock?
+    is not necessary, @scheduler = Scheduler.new if should_start?
+    the surrounding Ruby code checks
+[o] introduce job "vars", as in
+    http://stackoverflow.com/questions/18202848/how-to-have-a-variable-that-will-available-to-particular-scheduled-task-whenever
+    or job['key'] Job #[] and #[]=, as with Thread #[] #[]=
+    job-local variables #keys #key?
+[o] thread-safety for job-local variables?
+[x] discard past? discard_past => true or => "1d"
+    default would be discard_past => "1m" or scheduler freq * 2 ?
+    jobs would adjust their next_time until it fits the window...
+    ~~ discard past by default
+[o] expanded block/schedulable (it's "callable")
+    ```
+      scheduler.every '10m' do
+        def pre
+          return false if Backend.down?
+          # ...
+        end
+        def post
+          # ...
+        end
+        def trigger
+          puts "oh hai!"
+        end
+      end
+    ```
+    or something like that...
+    ...
+    OR accept a class (and instantiate it the first time)
+    ```
+      scheduler.every '10m', Class.new do
+        def call(job, time)
+          # ...
+        end
+      end
+    ```
+    the job contains the instance in its @callable
+[x] add spec case for corner case in Job#trigger (overlap vs reschedule) !!!
+[o] rethink job array vs job set for #scheduled?
+[x] introduce common parent class for EveryJob and IntervalJob
+[o] create spec/ at_job_spec.rb, repeat_job_spec.rb, cron_job_spec.rb, ...
+[x] ensure EveryJob do not schedule in the past (it's already like that)
+[o] CronLine#next_time should return a time with subseconds chopped off
+[o] drop min work threads setting?
+[o] thread pool something? Thread upper limit?
+[o] Rufus::Scheduler.singleton, Rufus::Scheduler.s
+[o] EveryJob#first_at= and IntervalJob#first_at= should alter @next_time
+[o] scheduler.schedule duration/time/cron ... for at/in/cron
+    (not every, nor interval)
+    scheduler.repeat time/cron ... for every/cron
+
+[o] :lockfile => x, timestamp, process_id, thread_id...
+    warning: have to clean up that file on exit... or does the scheduler
+    timestamps it?
+[ ] develop lockfile timestamp thinggy
+    ~ if the timestamp is too old (twice the default frequency?) then
+    lock [file] take over...
+    Is that really what we want all the time?
+
+[ ] idea: :mutex => x and :skip_on_mutex => true ?
+    would prevent blocking/waiting for the mutex to get available
+    :mutex => [ "mutex_name", true ]
+    :mutex => [ [ "mutex_name", true ], [ "other_mutex_name", false ] ]
+
+[ ] bring back EM (but only EM.defer ?) :defer => true (Job or Scheduler
+    or both option?)
+
+[ ] prepare a daemon, trust daemon-kit for that
+
+[ ] :if => lambda { |job, time| ... } why not?
+    :unless => lambda { ...
+    :block => lambda { ...
+    can help get the block themselves leaner
+    #
+    investigate guards for schedulables... def if_guard; ...; end
+
+[ ] scheduler.every '10', Class.new do
+      def call(job, time)
+        # might fail...
+      end
+      def on_error(err, job)
+        # catches...
+      end
+    end
 

data/lib/rufus/scheduler.rb

@@ -22,41 +22,517 @@
 # Made in Japan.
 #++
 
+require 'date' if RUBY_VERSION < '1.9.0'
+require 'time'
+require 'thread'
+require 'tzinfo'
+require 'fileutils'
 
-require 'rufus/sc/scheduler'
 
+module Rufus
 
-module Rufus::Scheduler
+  class Scheduler
 
-  # Starts and return a new instance of a PlainScheduler.
-  #
-  def self.new(opts={})
+    require 'rufus/scheduler/util'
+    require 'rufus/scheduler/jobs'
+    require 'rufus/scheduler/cronline'
+    require 'rufus/scheduler/job_array'
 
-    PlainScheduler.start_new(opts)
-  end
+    VERSION = '3.0.0'
+
+    #
+    # This error is thrown when the :timeout attribute triggers
+    #
+    class TimeoutError < StandardError; end
+
+    #MIN_WORK_THREADS = 7
+    MAX_WORK_THREADS = 35
+
+    attr_accessor :frequency
+    attr_reader :started_at
+    attr_reader :thread
+    attr_reader :thread_key
+    attr_reader :mutexes
+
+    #attr_accessor :min_work_threads
+    attr_accessor :max_work_threads
+
+    attr_accessor :stderr
+
+    attr_reader :work_queue
+
+    def initialize(opts={})
+
+      @opts = opts
+
+      @started_at = nil
+      @paused = false
+
+      @jobs = JobArray.new
+
+      @frequency = Rufus::Scheduler.parse(opts[:frequency] || 0.300)
+      @mutexes = {}
+
+      @work_queue = Queue.new
 
-  # A quick way to get a scheduler up an running
-  #
-  #   require 'rubygems'
-  #   s = Rufus::Scheduler.start_new
-  #
-  # If EventMachine is present and running will create an EmScheduler, else
-  # it will create a PlainScheduler instance.
-  #
-  def self.start_new(opts={})
-
-    if defined?(EM) and EM.reactor_running?
-      EmScheduler.start_new(opts)
-    else
-      PlainScheduler.start_new(opts)
+      #@min_work_threads = opts[:min_work_threads] || MIN_WORK_THREADS
+      @max_work_threads = opts[:max_work_threads] || MAX_WORK_THREADS
+
+      @stderr = $stderr
+
+      @thread_key = "rufus_scheduler_#{self.object_id}"
+
+      consider_lockfile || return
+
+      start
+    end
+
+    # Returns a singleton Rufus::Scheduler instance
+    #
+    def self.singleton(opts={})
+
+      @singleton ||= Rufus::Scheduler.new(opts)
+    end
+
+    # Alias for Rufus::Scheduler.singleton
+    #
+    def self.s(opts={}); singleton(opts); end
+
+    # Releasing the gem would probably require redirecting .start_new to
+    # .new and emit a simple deprecation message.
+    #
+    # For now, let's assume the people pointing at rufus-scheduler/master
+    # on GitHub know what they do...
+    #
+    def self.start_new
+
+      fail "this is rufus-scheduler 3.0, use .new instead of .start_new"
+    end
+
+    def shutdown(opt=nil)
+
+      @started_at = nil
+
+      jobs.each { |j| j.unschedule }
+
+      @work_queue.clear
+
+      if opt == :wait
+        join_all_work_threads
+      elsif opt == :kill
+        kill_all_work_threads
+      end
+
+      @lockfile.flock(File::LOCK_UN) if @lockfile
+    end
+
+    alias stop shutdown
+
+    def uptime
+
+      @started_at ? Time.now - @started_at : nil
+    end
+
+    def uptime_s
+
+      self.class.to_duration(uptime)
+    end
+
+    def join
+
+      @thread.join
+    end
+
+    def paused?
+
+      @paused
+    end
+
+    def pause
+
+      @paused = true
+    end
+
+    def resume
+
+      @paused = false
     end
-  end
 
-  # Returns true if the given string seems to be a cron string.
-  #
-  def self.is_cron_string(s)
+    #--
+    # scheduling methods
+    #++
+
+    def at(time, callable=nil, opts={}, &block)
+
+      do_schedule(:once, time, callable, opts, opts[:job], block)
+    end
+
+    def schedule_at(time, callable=nil, opts={}, &block)
+
+      do_schedule(:once, time, callable, opts, true, block)
+    end
+
+    def in(duration, callable=nil, opts={}, &block)
+
+      do_schedule(:once, duration, callable, opts, opts[:job], block)
+    end
+
+    def schedule_in(duration, callable=nil, opts={}, &block)
+
+      do_schedule(:once, duration, callable, opts, true, block)
+    end
+
+    def every(duration, callable=nil, opts={}, &block)
+
+      do_schedule(:every, duration, callable, opts, opts[:job], block)
+    end
+
+    def schedule_every(duration, callable=nil, opts={}, &block)
+
+      do_schedule(:every, duration, callable, opts, true, block)
+    end
+
+    def interval(duration, callable=nil, opts={}, &block)
+
+      do_schedule(:interval, duration, callable, opts, opts[:job], block)
+    end
+
+    def schedule_interval(duration, callable=nil, opts={}, &block)
+
+      do_schedule(:interval, duration, callable, opts, true, block)
+    end
+
+    def cron(cronline, callable=nil, opts={}, &block)
+
+      do_schedule(:cron, cronline, callable, opts, opts[:job], block)
+    end
+
+    def schedule_cron(cronline, callable=nil, opts={}, &block)
+
+      do_schedule(:cron, cronline, callable, opts, true, block)
+    end
+
+    def schedule(arg, callable=nil, opts={}, &block)
+
+      # TODO: eventually, spare one parse call
+
+      case Scheduler.parse(arg)
+        when CronLine then schedule_cron(arg, callable, opts, &block)
+        when Time then schedule_at(arg, callable, opts, &block)
+        else schedule_in(arg, callable, opts, &block)
+      end
+    end
+
+    def repeat(arg, callable=nil, opts={}, &block)
+
+      # TODO: eventually, spare one parse call
+
+      case Scheduler.parse(arg)
+        when CronLine then schedule_cron(arg, callable, opts, &block)
+        else schedule_every(arg, callable, opts, &block)
+      end
+    end
+
+    def unschedule(job_or_job_id)
+
+      job, job_id = fetch(job_or_job_id)
+
+      fail ArgumentError.new("no job found with id '#{job_id}'") unless job
+
+      job.unschedule if job
+    end
+
+    #--
+    # jobs methods
+    #++
+
+    # Returns all the scheduled jobs
+    # (even those right before re-schedule).
+    #
+    def jobs(opts={})
+
+      opts = { opts => true } if opts.is_a?(Symbol)
+
+      jobs = @jobs.to_a
+
+      if opts[:running]
+        jobs = jobs.select { |j| j.running? }
+      elsif ! opts[:all]
+        jobs = jobs.reject { |j| j.next_time.nil? || j.unscheduled_at }
+      end
+
+      tags = Array(opts[:tag] || opts[:tags]).collect { |t| t.to_s }
+      jobs = jobs.reject { |j| tags.find { |t| ! j.tags.include?(t) } }
+
+      jobs
+    end
 
-    s.match(/.+ .+ .+ .+ .+/) # well...
+    def at_jobs(opts={})
+
+      jobs(opts).select { |j| j.is_a?(Rufus::Scheduler::AtJob) }
+    end
+
+    def in_jobs(opts={})
+
+      jobs(opts).select { |j| j.is_a?(Rufus::Scheduler::InJob) }
+    end
+
+    def every_jobs(opts={})
+
+      jobs(opts).select { |j| j.is_a?(Rufus::Scheduler::EveryJob) }
+    end
+
+    def interval_jobs(opts={})
+
+      jobs(opts).select { |j| j.is_a?(Rufus::Scheduler::IntervalJob) }
+    end
+
+    def cron_jobs(opts={})
+
+      jobs(opts).select { |j| j.is_a?(Rufus::Scheduler::CronJob) }
+    end
+
+    def job(job_id)
+
+      @jobs[job_id]
+    end
+
+    # Returns true if this job is currently scheduled.
+    #
+    # Takes extra care to answer true if the job is a repeat job
+    # currently firing.
+    #
+    def scheduled?(job_or_job_id)
+
+      job, job_id = fetch(job_or_job_id)
+
+      !! (job && job.next_time != nil)
+    end
+
+    # Lists all the threads associated with this scheduler.
+    #
+    def threads
+
+      Thread.list.select { |t| t[thread_key] }
+    end
+
+    # Lists all the work threads (the ones actually running the scheduled
+    # block code)
+    #
+    # Accepts a query option, which can be set to:
+    # * :all (default), returns all the threads that are work threads
+    #   or are currently running a job
+    # * :active, returns all threads that are currenly running a job
+    # * :vacant, returns the threads that are not running a job
+    #
+    # If, thanks to :blocking => true, a job is scheduled to monopolize the
+    # main scheduler thread, that thread will get returned when :active or
+    # :all.
+    #
+    def work_threads(query=:all)
+
+      ts =
+        threads.select { |t|
+          t[:rufus_scheduler_job] || t[:rufus_scheduler_work_thread]
+        }
+
+      case query
+        when :active then ts.select { |t| t[:rufus_scheduler_job] }
+        when :vacant then ts.reject { |t| t[:rufus_scheduler_job] }
+        else ts
+      end
+    end
+
+    def running_jobs(opts={})
+
+      jobs(opts.merge(:running => true))
+    end
+
+    def on_error(job, err)
+
+      pre = err.object_id.to_s
+
+      stderr.puts("{ #{pre} rufus-scheduler intercepted an error:")
+      stderr.puts("  #{pre}   job:")
+      stderr.puts("  #{pre}     #{job.class} #{job.original.inspect} #{job.opts.inspect}")
+      stderr.puts("  #{pre}   error:")
+      stderr.puts("  #{pre}     #{err.object_id}")
+      stderr.puts("  #{pre}     #{err.class}")
+      stderr.puts("  #{pre}     #{err}")
+      err.backtrace.each do |l|
+        stderr.puts("  #{pre}       #{l}")
+      end
+      stderr.puts("} #{pre} .")
+
+    rescue => e
+
+      stderr.puts("failure in #on_error itself:")
+      stderr.puts(e.inspect)
+      stderr.puts(e.backtrace)
+
+    ensure
+
+      stderr.flush
+    end
+
+    protected
+
+    # Returns [ job, job_id ]
+    #
+    def fetch(job_or_job_id)
+
+      if job_or_job_id.respond_to?(:job_id)
+        [ job_or_job_id, job_or_job_id.job_id ]
+      else
+        [ job(job_or_job_id), job_or_job_id ]
+      end
+    end
+
+    def consider_lockfile
+
+      @lockfile = nil
+
+      return true unless f = @opts[:lockfile]
+
+      raise ArgumentError.new(
+        ":lockfile argument must be a string, not a #{f.class}"
+      ) unless f.is_a?(String)
+
+      FileUtils.mkdir_p(File.dirname(f))
+
+      f = File.new(f, File::RDWR | File::CREAT)
+      locked = f.flock(File::LOCK_NB | File::LOCK_EX)
+
+      return false unless locked
+
+      now = Time.now
+
+      f.print("pid: #{$$}, ")
+      f.print("scheduler.object_id: #{self.object_id}, ")
+      f.print("time: #{now}, ")
+      f.print("timestamp: #{now.to_f}")
+      f.flush
+
+      @lockfile = f
+
+      true
+    end
+
+    def terminate_all_jobs
+
+      jobs.each { |j| j.unschedule }
+
+      sleep 0.01 while running_jobs.size > 0
+    end
+
+    def join_all_work_threads
+
+      work_threads.size.times { @work_queue << :sayonara }
+
+      work_threads.each { |t| t.join }
+
+      @work_queue.clear
+    end
+
+    def kill_all_work_threads
+
+      work_threads.each { |t| t.kill }
+    end
+
+    #def free_all_work_threads
+    #
+    #  work_threads.each { |t| t.raise(KillSignal) }
+    #end
+
+    def start
+
+      @started_at = Time.now
+
+      @thread =
+        Thread.new do
+
+          while @started_at do
+
+            unschedule_jobs
+            trigger_jobs unless @paused
+            timeout_jobs
+
+            sleep(@frequency)
+          end
+        end
+
+      @thread[@thread_key] = true
+      @thread[:rufus_scheduler] = self
+      @thread[:name] = @opts[:thread_name] || "#{@thread_key}_scheduler"
+    end
+
+    def unschedule_jobs
+
+      @jobs.delete_unscheduled
+    end
+
+    def trigger_jobs
+
+      now = Time.now
+
+      @jobs.each(now) do |job|
+
+        job.trigger(now)
+      end
+    end
+
+    def timeout_jobs
+
+      work_threads(:active).each do |t|
+
+        job = t[:rufus_scheduler_job]
+        to = t[:rufus_scheduler_timeout]
+
+        next unless job && to
+          # thread might just have become inactive (job -> nil)
+
+        ts = t[:rufus_scheduler_time]
+        to = to.is_a?(Time) ? to : ts + to
+
+        next if to > Time.now
+
+        t.raise(Rufus::Scheduler::TimeoutError)
+      end
+    end
+
+    def do_schedule(job_type, t, callable, opts, return_job_instance, block)
+
+      raise RuntimeError.new(
+        'cannot schedule, scheduler is down or shutting down'
+      ) if @started_at == nil
+
+      callable, opts = nil, callable if callable.is_a?(Hash)
+      return_job_instance ||= opts[:job]
+
+      job_class =
+        case job_type
+          when :once
+            tt = Rufus::Scheduler.parse(t)
+            tt.is_a?(Time) ? AtJob : InJob
+          when :every
+            EveryJob
+          when :interval
+            IntervalJob
+          when :cron
+            CronJob
+        end
+
+      job = job_class.new(self, t, opts, block || callable)
+
+      raise ArgumentError.new(
+        "job frequency (#{job.frequency}) is higher than " +
+        "scheduler frequency (#{@frequency})"
+      ) if job.respond_to?(:frequency) && job.frequency < @frequency
+
+      @jobs.push(job)
+
+      return_job_instance ? job : job.job_id
+    end
   end
 end