sidetiq 0.3.7 → 0.4.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd23a1f597a8ea9f38f1a7d978c25aacb9881278
-  data.tar.gz: 38857c20bd446738e877befb5357da050d317f7b
+  metadata.gz: afa72994a97058431570f5d871d813841987f3d3
+  data.tar.gz: 481d72a325c1c96c24cd0f0f38a346fc85077abf
 SHA512:
-  metadata.gz: 4f3b2ac3341c8a3fa3f4b38ca192a8ff234f066d2fae8ebd2b0012393ca0fc418da7b64603caec2bf1aabf8fbed61200326fda4c74d0b6c3ac1422fa0fbc6c6f
-  data.tar.gz: 6024d079bea833aeef1d2b803bb32ed2f2dfe9108333bdaa6a9558ab28a07c3b67c52a38f5f34980ed4de29c4f988b50f78ed87a82b79481b3f57330f900639f
+  metadata.gz: ea02330ee1cceb86a386b2d47bb2b825462e549fbf236f411bf08c9b55172ec4d7073e5eaab6b9ebbabff1c2280c330ad0247014bcbcc5625ed6d4651ab0430f
+  data.tar.gz: 5989c73b9f8e9c203b18ffa22dcedc2f92e06eeceafa88f6e301e08a0d8c631d6ceeb6b47fa747fc85c528711208a67e4a6d29b53993e0df04ca342271a269d3

data/.travis.yml

@@ -1,4 +1,5 @@
 language: ruby
+script: rake test
 branches:
   only:
     - master

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+0.4.0
+-----
+
+- Show job history in web extension.
+- Integrate with Sidekiq's exception handling/reporting in critical parts.
+- Store more detailed lock metadata.
+- Remove stray 'thead' from ERB template.
+- Store scheduled worker history in Redis.
+- Use a Celluloid pool of scheduling handlers to run calculations in parallel.
+- Use Celluloid actors instead of plain threads.
+- Fix to work with workers with one optional argument. [nata79]
+- Refactor top-level namespace methods into separate modules.
+- Add Procfile-based example code to boot Sidekiq and the web frontend
+  simultaneuously.
+- Experimental watcher worker to remove invalid locks.
+
 0.3.7
 -----
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2013, Tobias Svensson <tobias@musicglue.com>
+Copyright (c) 2012-2013, Tobias Svensson <tob@tobiassvensson.co.uk>
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -21,4 +21,4 @@ DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -32,9 +32,11 @@ Overview
 
 Sidetiq provides a simple API for defining recurring workers for Sidekiq.
 
+- Cuncurrency using Celluloid actors.
+
 - Flexible DSL based on [ice_cube](http://seejohnrun.github.com/ice_cube/)
 
-- Sidetiq uses a locking mechanism (based on `setnx` and `pexpire`) internally
+- Sidetiq uses a locking mechanism (based on watch/multi/psetex/exec) internally
   so Sidetiq clocks can run in each Sidekiq process without interfering with
   each other (tested with sub-second polling of scheduled jobs by Sidekiq and
   Sidetiq clock rates above 100hz).
@@ -46,6 +48,7 @@ Dependencies
 ------------
 
 - [Sidekiq](http://mperham.github.com/sidekiq/)
+- [Celluloid](http://celluloid.io/) (shared with Sidekiq)
 - [ice_cube](http://seejohnrun.github.com/ice_cube/)
 
 <a name='section_Installation'></a>
@@ -136,17 +139,8 @@ class MyWorker
 end
 ```
 
-To start Sidetiq, simply call `Sidetiq::Clock.start!` in a server specific
-configuration block:
-
-```ruby
-Sidekiq.configure_server do |config|
-  Sidetiq::Clock.start!
-end
-```
-
-Additionally, Sidetiq includes a middleware that will check if the clock
-thread is still alive and restart it if necessary.
+If Sidekiq is running in server-mode, Sidekiq will handle recurring
+jobs automatically.
 
 <a name='section_Backfills''></a>
 Backfills
@@ -180,10 +174,6 @@ Configuration
 
 ```ruby
 Sidetiq.configure do |config|
-  # Thread priority of the clock thread (default: Thread.main.priority as
-  # defined when Sidetiq is loaded).
-  config.priority = 2
-
   # Clock tick resolution in seconds (default: 1).
   config.resolution = 0.5
 
@@ -192,6 +182,9 @@ Sidetiq.configure do |config|
 
   # When `true` uses UTC instead of local times (default: false)
   config.utc = false
+
+  # Scheduling handler pool size (default: number of CPUs)
+  config.handler_pool_size = 5
 end
 ```
 <a name='section_Configuration_Logging'></a>

data/Rakefile

@@ -1,8 +1,3 @@
-require 'bundler/gem_tasks'
-require 'rake/testtask'
+Dir["tasks/**/*.task"].each { |task| load task }
 
-Rake::TestTask.new do |t|
-  t.pattern = 'test/**/test_*.rb'
-end
-
-task default: :test
+task :default => :test

data/examples/Procfile

@@ -0,0 +1,2 @@
+sidekiq: bundle exec sidekiq -r ./server.rb
+web: bundle exec rackup

data/examples/config.ru

@@ -0,0 +1,15 @@
+require 'sidekiq'
+require 'sidetiq'
+
+require 'sidekiq/web'
+require 'sidetiq/web'
+require 'sidetiq/lock/watcher'
+
+require './workers/simple.rb'
+require './workers/failing.rb'
+
+Sidekiq.configure_client do |config|
+  config.redis = { :size => 1 }
+end
+
+run Sidekiq::Web

data/examples/server.rb

@@ -0,0 +1,17 @@
+# Run with `sidekiq -r /path/to/simple.rb`
+
+require 'sidekiq'
+require 'sidetiq'
+require 'sidetiq/lock/watcher'
+
+require_relative 'workers/simple'
+require_relative 'workers/failing'
+
+Sidekiq.logger.level = Logger::DEBUG
+
+Sidekiq.options[:poll_interval] = 1
+
+Sidekiq.configure_server do |config|
+  Sidetiq.clock.start!
+end
+

data/examples/workers/failing.rb

@@ -0,0 +1,13 @@
+class Failing
+  include Sidekiq::Worker
+  include Sidetiq::Schedulable
+
+  recurrence { secondly }
+
+  def perform(*args)
+    if rand(5) == 0
+      raise "This didn't go well."
+    end
+  end
+end
+

data/examples/workers/simple.rb

@@ -0,0 +1,10 @@
+class Simple
+  include Sidekiq::Worker
+  include Sidetiq::Schedulable
+
+  recurrence { secondly }
+
+  def perform(*args)
+  end
+end
+

data/lib/sidetiq/actor/clock.rb

@@ -0,0 +1,31 @@
+module Sidetiq
+  module Actor
+    class Clock < Sidetiq::Clock
+      include Sidetiq::Actor
+      include Sidekiq::ExceptionHandler
+
+      # Public: Starts the clock loop.
+      def start!
+        debug "Sidetiq::Clock looping ..."
+        loop!
+      end
+
+      private
+
+      def loop!
+        after([time { tick }, 0].max) do
+          loop!
+        end
+      rescue StandardError => e
+        handle_exception(e, context: 'Sidetiq::Clock#loop!')
+        retry
+      end
+
+      def time
+        start = gettime
+        yield
+        Sidetiq.config.resolution - (gettime.to_f - start.to_f)
+      end
+    end
+  end
+end

data/lib/sidetiq/actor/handler.rb

@@ -0,0 +1,7 @@
+module Sidetiq
+  module Actor
+    class Handler < Sidetiq::Handler
+      include Sidetiq::Actor
+    end
+  end
+end

data/lib/sidetiq/actor.rb

@@ -0,0 +1,38 @@
+module Sidetiq
+  module Actor
+    def self.included(base)
+      base.__send__(:include, Celluloid)
+      base.finalizer :sidetiq_finalizer
+    end
+
+    def initialize(*args, &block)
+      log_call "initialize"
+
+      super
+
+      # Link to Sidekiq::Manager when running in server-mode. In most
+      # cases the supervisor is booted before Sidekiq has launched
+      # fully, so defer this.
+      if Sidekiq.server?
+        after(0.1) { link_to_sidekiq_manager }
+      end
+    end
+
+    private
+
+    def sidetiq_finalizer
+      log_call "shutting down ..."
+    end
+
+    def link_to_sidekiq_manager
+      Sidekiq::CLI.instance.launcher.manager.link(current_actor)
+    rescue NoMethodError
+      warn "Can't link #{self.class.name}. Sidekiq::Manager not running. Retrying ..."
+      after(1) { link_to_sidekiq_manager }
+    end
+
+    def log_call(call)
+      info "#{self.class.name} id: #{object_id} #{call}"
+    end
+  end
+end

data/lib/sidetiq/api.rb

@@ -0,0 +1,109 @@
+module Sidetiq
+  # Public: Sidetiq API methods.
+  module API
+    # Public: Returns an Array of workers including Sidetiq::Schedulable.
+    def workers
+      schedules.keys
+    end
+
+    # Public: Returns a Hash of Sidetiq::Schedule instances.
+    def schedules
+      clock.schedules.dup
+    end
+
+    # Public: Currently scheduled recurring jobs.
+    #
+    # worker - A Sidekiq::Worker class or String of the class name (optional)
+    # block  - An optional block that can be given to which each
+    #          Sidekiq::SortedEntry instance corresponding to a scheduled job will
+    #          be yielded.
+    #
+    # Examples
+    #
+    #   Sidetiq.scheduled
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.scheduled(MyWorker)
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.scheduled("MyWorker")
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.scheduled do |job|
+    #     # do stuff ...
+    #   end
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.scheduled(MyWorker) do |job|
+    #     # do stuff ...
+    #   end
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.scheduled("MyWorker") do |job|
+    #     # do stuff ...
+    #   end
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    # Yields each Sidekiq::SortedEntry instance.
+    # Returns an Array of Sidekiq::SortedEntry objects.
+    def scheduled(worker = nil, &block)
+      filter_set(Sidekiq::ScheduledSet.new, worker, &block)
+    end
+
+    # Public: Recurring jobs currently scheduled for retries.
+    #
+    # worker - A Sidekiq::Worker class or String of the class name (optional)
+    # block  - An optional block that can be given to which each
+    #          Sidekiq::SortedEntry instance corresponding to a scheduled job will
+    #          be yielded.
+    #
+    # Examples
+    #
+    #   Sidetiq.retries
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.retries(MyWorker)
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.retries("MyWorker")
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.retries do |job|
+    #     # do stuff ...
+    #   end
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.retries(MyWorker) do |job|
+    #     # do stuff ...
+    #   end
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    #   Sidetiq.retries("MyWorker") do |job|
+    #     # do stuff ...
+    #   end
+    #   # => [#<Sidekiq::SortedEntry>, ...]
+    #
+    # Yields each Sidekiq::SortedEntry instance.
+    # Returns an Array of Sidekiq::SortedEntry objects.
+    def retries(worker = nil, &block)
+      filter_set(Sidekiq::RetrySet.new, worker, &block)
+    end
+
+    private
+
+    def filter_set(set, worker, &block)
+      worker = worker.constantize if worker.kind_of?(String)
+
+      jobs = set.select do |job|
+        klass = job.klass.constantize
+        ret = klass.include?(Schedulable)
+        ret = ret && klass == worker if worker
+        ret
+      end
+
+      jobs.each(&block) if block_given?
+
+      jobs
+    end
+  end
+end

data/lib/sidetiq/clock.rb

@@ -1,26 +1,11 @@
 module Sidetiq
-  configure do |config|
-    config.priority = Thread.main.priority
-    config.resolution = 1
-    config.lock_expire = 1000
-    config.utc = false
-  end
-
   # Public: The Sidetiq clock.
   class Clock
-    include Singleton
-    include MonitorMixin
+    include Logging
 
     # Internal: Returns a hash of Sidetiq::Schedule instances.
     attr_reader :schedules
 
-    # Internal: Returns the clock thread.
-    attr_reader :thread
-
-    def self.method_missing(meth, *args, &block) # :nodoc:
-      instance.__send__(meth, *args, &block)
-    end
-
     def initialize # :nodoc:
       super
       @schedules = {}
@@ -50,18 +35,8 @@ module Sidetiq
     # Returns a hash of Sidetiq::Schedule instances.
     def tick
       tick = gettime
-      mon_synchronize do
-        schedules.each do |worker, sched|
-          Lock.new(worker).synchronize do |redis|
-            if sched.backfill? && (last = worker.last_scheduled_occurrence) > 0
-              last = Sidetiq.config.utc ? Time.at(last).utc : Time.at(last)
-              sched.occurrences_between(last + 1, tick).each do |past_t|
-                enqueue(worker, past_t, redis)
-              end
-            end
-            enqueue(worker, sched.next_occurrence(tick), redis)
-          end if sched.schedule_next?(tick)
-        end
+      schedules.each do |worker, sched|
+        Sidetiq.handler.dispatch(worker,sched, tick)
       end
     end
 
@@ -76,96 +51,6 @@ module Sidetiq
     def gettime
       Sidetiq.config.utc ? Time.now.utc : Time.now
     end
-
-    # Public: Starts the clock unless it is already running.
-    #
-    # Examples
-    #
-    #   start!
-    #   # => Thread
-    #
-    # Returns the Thread instance of the clock thread.
-    def start!
-      return if ticking?
-
-      Sidetiq.logger.info "Sidetiq::Clock start"
-
-      @thread = Thread.start { clock { tick } }
-      @thread.abort_on_exception = true
-      @thread.priority = Sidetiq.config.priority
-      @thread
-    end
-
-    # Public: Stops the clock if it is running.
-    #
-    # Examples
-    #
-    #   stop!
-    #   # => nil
-    #
-    # Returns nil.
-    def stop!
-      if ticking?
-        @thread.kill
-        Sidetiq.logger.info "Sidetiq::Clock stop"
-      end
-    end
-
-    # Public: Returns the status of the clock.
-    #
-    # Examples
-    #
-    #   ticking?
-    #   # => false
-    #
-    #   start!
-    #   ticking?
-    #   # => true
-    #
-    # Returns true or false.
-    def ticking?
-      @thread && @thread.alive?
-    end
-
-    private
-
-    def enqueue(worker, time, redis)
-      key      = "sidetiq:#{worker.name}"
-      time_f   = time.to_f
-      next_run = (redis.get("#{key}:next") || -1).to_f
-
-      if next_run < time_f
-        Sidetiq.logger.info "Sidetiq::Clock enqueue #{worker.name} (at: #{time_f}) (last: #{next_run})"
-
-        redis.mset("#{key}:last", next_run, "#{key}:next", time_f)
-
-        case worker.instance_method(:perform).arity
-        when 0
-          worker.perform_at(time)
-        when 1
-          worker.perform_at(time, next_run)
-        else
-          worker.perform_at(time, next_run, time_f)
-        end
-      end
-    end
-
-    def clock
-      loop do
-        sleep_time = time { yield }
-
-        if sleep_time > 0
-          Thread.pass
-          sleep sleep_time
-        end
-      end
-    end
-
-    def time
-      start = gettime
-      yield
-      Sidetiq.config.resolution - (gettime.to_f - start.to_f)
-    end
   end
 end
 

data/lib/sidetiq/config.rb

@@ -22,5 +22,13 @@ module Sidetiq
       @config ||= OpenStruct.new
     end
   end
+
+  configure do |config|
+    config.worker_history = 50
+    config.resolution = 1
+    config.lock_expire = 1000
+    config.utc = false
+    config.handler_pool_size = nil
+  end
 end
 

data/lib/sidetiq/handler.rb

@@ -0,0 +1,50 @@
+module Sidetiq
+  class Handler
+    include Logging
+    include Sidekiq::ExceptionHandler
+
+    def dispatch(worker, sched, tick)
+      return unless sched.schedule_next?(tick)
+
+      Lock::Redis.new(worker).synchronize do |redis|
+        if sched.backfill? && (last = worker.last_scheduled_occurrence) > 0
+          last = Sidetiq.config.utc ? Time.at(last).utc : Time.at(last)
+          sched.occurrences_between(last + 1, tick).each do |past_t|
+            enqueue(worker, past_t, redis)
+          end
+        end
+
+        enqueue(worker, sched.next_occurrence(tick), redis)
+      end
+    rescue StandardError => e
+      handle_exception(e, context: "Sidetiq::Handler#dispatch")
+      raise e
+    end
+
+    private
+
+    def enqueue(worker, time, redis)
+      key      = "sidetiq:#{worker.name}"
+      time_f   = time.to_f
+      next_run = (redis.get("#{key}:next") || -1).to_f
+
+      if next_run < time_f
+        info "Enqueue: #{worker.name} (at: #{time_f}) (last: #{next_run})"
+
+        redis.mset("#{key}:last", next_run, "#{key}:next", time_f)
+
+        case worker.instance_method(:perform).arity.abs
+        when 0
+          worker.perform_at(time)
+        when 1
+          worker.perform_at(time, next_run)
+        else
+          worker.perform_at(time, next_run, time_f)
+        end
+      end
+    rescue StandardError => e
+      handle_exception(e, context: "Sidetiq::Handler#enqueue")
+      raise e
+    end
+  end
+end

data/lib/sidetiq/lock/meta_data.rb

@@ -0,0 +1,53 @@
+module Sidetiq
+  module Lock
+    class MetaData
+      OWNER = "#{Socket.gethostname}:#{Process.pid}"
+
+      attr_accessor :owner, :timestamp, :key
+
+      class << self
+        include Sidekiq::ExceptionHandler
+
+        def for_new_lock(key)
+          new(owner: OWNER, timestamp: Sidetiq.clock.gettime.to_f, key: key)
+        end
+
+        def from_json(json = "")
+          # Avoid TypeError when nil is passed to JSON.parse.
+          json = "" if json.nil?
+
+          hash = JSON.parse(json, symbolize_names: true)
+          new(hash)
+        rescue JSON::ParserError => e
+          if json != ""
+            # Looks like garbage lock metadata, so report it.
+            handle_exception(e, context: "Garbage lock meta data detected: #{json}")
+          end
+
+          new
+        end
+      end
+
+      def initialize(hash = {})
+        @owner = hash[:owner]
+        @timestamp = hash[:timestamp]
+        @key = hash[:key]
+      end
+
+      def pttl
+        Sidekiq.redis { |r| r.pttl(key) }
+      end
+
+      def to_json
+        instance_variables.each_with_object({}) do |var, hash|
+          hash[var.to_s.delete("@")] = instance_variable_get(var)
+        end.to_json
+      end
+
+      def to_s
+        "Sidetiq::Lock on #{key} set at #{timestamp} by #{owner}"
+      end
+    end
+  end
+end
+