sidekiq 0.5.1 → 0.6.0

data/.gitignore

@@ -0,0 +1,5 @@
+tags
+Gemfile.lock
+*.swp
+dump.rdb
+.rbx

data/.rvmrc

@@ -1,3 +1,4 @@
-export RUBYOPT="-Ilib"
+export RUBYOPT="-Ilib:test"
 export JRUBY_OPTS="--1.9"
 rvm use 1.9.3@sidekiq --create
+#rvm use jruby@sidekiq --create

data/Changes.md

@@ -1,3 +1,14 @@
+0.6.0
+-----------
+
+- Resque-compatible processing stats in redis (mperham)
+- Simple client testing support in sidekiq/testing (mperham)
+- Plain old Ruby support via the -r cli flag (mperham)
+- Refactored middleware support, introducing ability to add client-side middleware (ryanlecompte)
+- Added middleware for ignoring duplicate jobs (ryanlecompte)
+- Added middleware for displaying jobs in resque-web dashboard (maxjustus)
+- Added redis namespacing support (maxjustus)
+
 0.5.1
 -----------
 

data/LICENSE

@@ -1,9 +1,16 @@
 Copyright (c) Mike Perham
 
-Sidekiq is open source licensed under the GPLv3 for non-commerical use:
+You may choose to license Sidekiq in one of two ways:
+
+- Open Source under the GPLv3
 
   Please see <http://www.gnu.org/licenses/gpl-3.0.html> for license text.
 
-To purchase a commercial license, you must pledge $50 to my Pledgie
-fund at http://www.pledgie.com/campaigns/16623.  The commercial license can
-be found in COMM-LICENSE.
+- Commercial License
+
+  The Sencha commercial software license in COMM-LICENSE.  You must pledge
+  $50 to my Pledgie fund at http://www.pledgie.com/campaigns/16623 to use
+  this license.
+
+This commercial license is just meant to help support my OSS work.  If
+you do buy the commercial licensing option, thank you for your support!

data/README.md

@@ -47,13 +47,7 @@ Please see the [sidekiq wiki](https://github.com/mperham/sidekiq/wiki) for more
 License
 -----------------
 
-sidekiq is GPLv3 licensed for **non-commercial use only**.  For a commercial
-license, you must give $50 to my [Pledgie campaign](http://www.pledgie.com/campaigns/16623).
-Considering the hundreds of hours I've spent writing OSS, I hope you
-think this is a reasonable price.  BTW, the commercial license is in
-COMM-LICENSE and is the [Sencha commercial license v1.10](http://www.sencha.com/legal/sencha-commercial-software-license-agreement/) with the Support (section 11) terms removed.
-Support is provided through GitHub Issues.  You are welcome to try the
-software for 2 weeks.
+Please see LICENSE for licensing details.
 
 <a href='http://www.pledgie.com/campaigns/16623'><img alt='Click here to lend your support to Open Source and make a donation at www.pledgie.com !' src='http://www.pledgie.com/campaigns/16623.png?skin_name=chrome' border='0' /></a>
 

data/TODO.md

@@ -1,4 +1,3 @@
-- monitoring stats
 - resque-ui-esque web ui
 - graceful shutdown (ideally Celluloid will provide this)
 - monit/god/etc example scripts

data/bin/sidekiq

@@ -11,9 +11,11 @@ end
 
 begin
   cli = Sidekiq::CLI.new
+  cli.parse
   cli.run
 rescue => e
   raise e if $DEBUG
   STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
   exit 1
 end

data/examples/por.rb

@@ -0,0 +1,17 @@
+require 'sidekiq'
+
+# Start up sidekiq via
+# ./bin/sidekiq -r ./examples/por.rb
+# and then you can open up an IRB session like so:
+# irb -r ./examples/por.rb
+# where you can then say
+# PlainOldRuby.perform_async "like a dog", 3
+#
+class PlainOldRuby
+  include Sidekiq::Worker
+
+  def perform(how_hard="super hard", how_long=1)
+    sleep how_long
+    puts "Workin' #{how_hard}"
+  end
+end

data/examples/sinkiq.rb

@@ -0,0 +1,57 @@
+# Make sure you have Sinatra installed, then start sidekiq with
+# ./bin/sidekiq -r ./examples/sinkiq.rb
+# Simply run Sinatra with
+# ruby examples/sinkiq.rb
+# and then browse to http://localhost:4567
+#
+require 'sinatra'
+require 'sidekiq/worker'
+$redis = Sidekiq::RedisConnection.create
+
+class SinatraWorker
+  include Sidekiq::Worker
+
+  def perform(msg="lulz you forgot a msg!")
+    $redis.lpush("sinkiq-example-messages", msg)      
+  end
+end
+
+get '/' do
+  @failed = $redis.get('stat:failed')
+  @processed = $redis.get('stat:processed')
+  @messages = $redis.lrange('sinkiq-example-messages', 0, -1)
+  erb :index
+end
+
+post '/msg' do
+  SinatraWorker.perform_async params[:msg]
+  redirect to('/')
+end
+
+__END__
+
+@@ layout
+<html>
+  <head>
+    <title>Sinatra + Sidekiq</title>
+    <body>
+      <%= yield %>
+    </body>
+</html>
+
+@@ index
+  <h1>Sinata + Sidekiq Example</h1>
+  <h2>Failed: <%= @failed %></h2>
+  <h2>Processed: <%= @processed %></h2>
+
+  <form method="post" action="/msg">
+    <input type="text" name="msg">
+    <input type="submit" value="Add Message">
+  </form>
+
+  <a href="/">Refresh page</a>
+
+  <h3>Messages</h3>
+  <% @messages.each do |msg| %>
+    <p><%= msg %></p>
+  <% end %>

data/lib/sidekiq.rb

@@ -1,4 +1,4 @@
 require 'sidekiq/version'
 require 'sidekiq/client'
 require 'sidekiq/worker'
-require 'sidekiq/rails' if defined?(Rails)
+require 'sidekiq/rails' if defined?(::Rails)

data/lib/sidekiq/cli.rb

@@ -1,34 +1,50 @@
+trap 'INT' do
+  # Handle Ctrl-C in JRuby like MRI
+  # http://jira.codehaus.org/browse/JRUBY-4637
+  Thread.main.raise Interrupt
+end
+
+trap 'TERM' do
+  # Heroku sends TERM and then waits 10 seconds for process to exit.
+  Thread.main.raise Interrupt
+end
+
 require 'optparse'
 require 'sidekiq/version'
 require 'sidekiq/util'
-require 'sidekiq/client'
+require 'sidekiq/redis_connection'
 require 'sidekiq/manager'
-require 'connection_pool'
 
 module Sidekiq
   class CLI
     include Util
 
+    # Used for CLI testing
+    attr_accessor :options, :code
+
     def initialize
-      parse_options
-      validate!
-      boot_rails
+      @code = nil
     end
 
-    FOREVER = 2_000_000_000
+    def parse(args=ARGV)
+      Sidekiq::Util.logger
+      parse_options(args)
+      validate!
+      boot_system
+    end
 
     def run
-      ::Sidekiq::Client.redis = ConnectionPool.new { Redis.connect(:url => @options[:server]) }
-      manager = Sidekiq::Manager.new(@options[:server], @options)
+      Sidekiq::Manager.redis = RedisConnection.create(:url => @options[:server], :namespace => @options[:namespace])
+      manager = Sidekiq::Manager.new(@options)
       begin
-        log 'Starting processing, hit Ctrl-C to stop'
+        logger.info 'Starting processing, hit Ctrl-C to stop'
         manager.start!
         # HACK need to determine how to pause main thread while
         # waiting for signals.
-        sleep FOREVER
+        sleep
       rescue Interrupt
         # TODO Need clean shutdown support from Celluloid
-        log 'Shutting down, pausing 5 seconds to let workers finish...'
+        logger.info 'Shutting down, pausing 5 seconds to let workers finish...'
         manager.stop!
         manager.wait(:shutdown)
       end
@@ -36,54 +52,76 @@ module Sidekiq
 
     private
 
-    def boot_rails
-      ENV['RAILS_ENV'] = @options[:environment]
-      require File.expand_path("#{@options[:rails]}/config/environment.rb")
-      ::Rails.application.eager_load!
+    def die(code)
+      exit(code)
     end
 
-    def validate!
-      $DEBUG = @options[:verbose]
+    def detected_environment
+      @options[:environment] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+
+    def boot_system
+      ENV['RACK_ENV'] = ENV['RAILS_ENV'] = detected_environment
+
+      raise ArgumentError, "#{@options[:require]} does not exist" if !File.exist?(@options[:require])
 
-      if !File.exist?("#{@options[:rails]}/config/boot.rb")
-        log "========== Please point sidekiq to a Rails 3 application =========="
-        log @parser
-        exit(1)
+      if File.directory?(@options[:require])
+        require File.expand_path("#{@options[:require]}/config/environment.rb")
+        ::Rails.application.eager_load!
+      else
+        require @options[:require]
       end
     end
 
-    def parse_options(argv=ARGV)
+    def validate!
+      @options[:queues] << 'default' if @options[:queues].empty?
+      @options[:queues].shuffle!
+
+      if !File.exist?(@options[:require]) ||
+         (File.directory?(@options[:require]) && !File.exist?("#{@options[:require]}/config/application.rb"))
+        logger.info "=================================================================="
+        logger.info "  Please point sidekiq to a Rails 3 application or a Ruby file    "
+        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "=================================================================="
+        logger.info @parser
+        die(1)
+      end
+    end
+
+    def parse_options(argv)
       @options = {
-        :verbose => false,
-        :queues => ['default'],
+        :queues => [],
         :processor_count => 25,
-        :server => ENV['REDISTOGO_URL'] || 'redis://localhost:6379/0',
-        :rails => '.',
-        :environment => 'production',
+        :require => '.',
+        :environment => nil,
       }
 
       @parser = OptionParser.new do |o|
         o.on "-q", "--queue QUEUE,WEIGHT", "Queue to process, with optional weight" do |arg|
           (q, weight) = arg.split(",")
-          (weight || 1).times do
+          (weight || 1).to_i.times do
             @options[:queues] << q
           end
         end
 
         o.on "-v", "--verbose", "Print more verbose output" do
-          @options[:verbose] = true
+          Sidekiq::Util.logger.level = Logger::DEBUG
+        end
+
+        o.on "-n", "--namespace NAMESPACE", "namespace worker queues are under" do |arg|
+          @options[:namespace] = arg
         end
 
         o.on "-s", "--server LOCATION", "Where to find Redis" do |arg|
           @options[:server] = arg
         end
 
-        o.on '-e', '--environment ENV', "Rails application environment" do |arg|
+        o.on '-e', '--environment ENV', "Application environment" do |arg|
           @options[:environment] = arg
         end
 
-        o.on '-r', '--rails PATH', "Location of Rails application with workers" do |arg|
-          @options[:rails] = arg
+        o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
+          @options[:require] = arg
         end
 
         o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
@@ -93,8 +131,8 @@ module Sidekiq
 
       @parser.banner = "sidekiq [options]"
       @parser.on_tail "-h", "--help", "Show help" do
-        log @parser
-        exit 1
+        logger.info @parser
+        die 1
       end
       @parser.parse!(argv)
     end

data/lib/sidekiq/client.rb

@@ -1,30 +1,53 @@
 require 'multi_json'
 require 'redis'
 
+require 'sidekiq/redis_connection'
+require 'sidekiq/middleware/chain'
+require 'sidekiq/middleware/client/resque_web_compatibility'
+require 'sidekiq/middleware/client/unique_jobs'
+
 module Sidekiq
   class Client
 
-    def self.redis
-      @redis ||= begin
-        # autoconfig for Heroku
-        hash = {}
-        hash[:url] = ENV['REDISTOGO_URL'] if ENV['REDISTOGO_URL']
-        Redis.connect(hash)
+    def self.middleware
+      @middleware ||= begin
+        m = Middleware::Chain.new
+        m.register do
+          use Middleware::Client::UniqueJobs, Client.redis
+          use Middleware::Client::ResqueWebCompatibility, Client.redis
+        end
+        m
       end
     end
 
+    def self.queues
+      @queues ||= {}
+    end
+
+    def self.redis
+      @redis ||= RedisConnection.create
+    end
+
     def self.redis=(redis)
       @redis = redis
     end
 
     # Example usage:
     # Sidekiq::Client.push('my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
-    def self.push(queue='default', item)
+    def self.push(queue=nil, item)
       raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeClass, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
       raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
 
+      queue = queue || queues[item['class'].to_s] || 'default'
+
       item['class'] = item['class'].to_s if !item['class'].is_a?(String)
-      redis.rpush("queue:#{queue}", MultiJson.encode(item))
+
+      pushed = false
+      middleware.invoke(item, queue) do
+        redis.rpush("queue:#{queue}", MultiJson.encode(item))
+        pushed = true
+      end
+      pushed
     end
 
     # Please use .push if possible instead.
@@ -33,16 +56,10 @@ module Sidekiq
     #
     #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
     #
-    # Messages are enqueued to the 'default' queue.  Optionally,
-    # MyWorker can define a queue class method:
-    #
-    #   def self.queue
-    #     'my_queue'
-    #   end
+    # Messages are enqueued to the 'default' queue.
     #
     def self.enqueue(klass, *args)
-      queue = (klass.respond_to?(:queue) && klass.queue) || 'default'
-      push(queue, { 'class' => klass.name, 'args' => args })
+      push(nil, { 'class' => klass.name, 'args' => args })
     end
   end
 end

data/lib/sidekiq/manager.rb

@@ -18,22 +18,20 @@ module Sidekiq
 
     trap_exit :processor_died
 
-    def initialize(location, options={})
-      log "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{location}"
-      verbose options.inspect
+    class << self
+      attr_accessor :redis
+    end
+
+    def initialize(options={})
+      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis.client.location}"
+      logger.debug { options.inspect }
       @count = options[:processor_count] || 25
       @queues = options[:queues]
-      @queue_idx = 0
-      @queues_size = @queues.size
-      @redis = Redis.connect(:url => location)
       @done_callback = nil
 
       @done = false
       @busy = []
-      @ready = []
-      @count.times do
-        @ready << Processor.new_link(current_actor)
-      end
+      @ready = @count.times.map { Processor.new_link(current_actor) }
     end
 
     def stop
@@ -44,35 +42,38 @@ module Sidekiq
       after(5) do
         signal(:shutdown)
       end
+
+      redis.with_connection do |conn|
+        conn.smembers('workers').each do |name|
+          conn.srem('workers', name) if name =~ /:#{Process.pid}:/
+        end
+      end
     end
 
     def start
       dispatch(true)
     end
 
-    def when_done
-      @done_callback = Proc.new
+    def when_done(&blk)
+      @done_callback = blk
     end
 
     def processor_done(processor)
-      @done_callback.call(processor) if @done_callback
-      @busy.delete(processor)
-      if stopped?
-        processor.terminate
-      else
-        @ready << processor
+      watchdog('sidekiq processor_done crashed!') do
+        @done_callback.call(processor) if @done_callback
+        @busy.delete(processor)
+        if stopped?
+          processor.terminate
+        else
+          @ready << processor
+        end
+        dispatch
       end
-      dispatch
     end
 
     def processor_died(processor, reason)
       @busy.delete(processor)
 
-      if reason
-        log "Processor death: #{reason}"
-        log reason.backtrace.join("\n")
-      end
-
       unless stopped?
         @ready << Processor.new_link(current_actor)
         dispatch
@@ -81,13 +82,12 @@ module Sidekiq
 
     private
 
-    def find_work(queue_idx)
-      current_queue = @queues[queue_idx]
-      msg = @redis.lpop("queue:#{current_queue}")
+    def find_work(queue)
+      msg = redis.lpop("queue:#{queue}")
       if msg
         processor = @ready.pop
         @busy << processor
-        processor.process! MultiJson.decode(msg)
+        processor.process!(MultiJson.decode(msg), queue)
       end
       !!msg
     end
@@ -96,32 +96,22 @@ module Sidekiq
       watchdog("Fatal error in sidekiq, dispatch loop died") do
         return if stopped?
 
-        # Our dispatch loop
-        # Loop through the queues, looking for a message in each.
-        queue_idx = 0
-        found = false
+        # Dispatch loop
         loop do
-          # return so that we don't dispatch again until processor_done
-          break verbose('no processors') if @ready.size == 0
-
-          found ||= find_work(queue_idx)
-          queue_idx += 1
-
-          # if we find no messages in any of the queues, we can break
-          # out of the loop.  Otherwise we loop again.
-          lastq = (queue_idx % @queues.size == 0)
-          if lastq && !found
-            verbose('nothing to process'); break
-          elsif lastq
-            queue_idx = 0
-            found = false
+          break logger.debug('no processors') if @ready.empty?
+          found = false
+          @ready.size.times do
+            found ||= find_work(@queues.sample)
           end
+          break logger.debug('nothing to process') unless found
         end
 
         # This is the polling loop that ensures we check Redis every
         # second for work, even if there was nothing to do this time
         # around.
-        after(1) { verbose('ping'); dispatch(schedule) } if schedule
+        after(1) do
+          dispatch(schedule)
+        end if schedule
       end
     end