sidekiq 0.6.0 → 0.7.0

data/lib/sidekiq/middleware/client/resque_web_compatibility.rb

@@ -2,12 +2,9 @@ module Sidekiq
   module Middleware
     module Client
       class ResqueWebCompatibility
-        def initialize(redis)
-          @redis = redis
-        end
 
         def call(item, queue)
-          @redis.sadd('queues', queue)
+          Sidekiq.redis.sadd('queues', queue)
           yield
         end
 

data/lib/sidekiq/middleware/client/unique_jobs.rb

@@ -1,3 +1,4 @@
+require 'multi_json'
 require 'digest'
 
 module Sidekiq
@@ -6,24 +7,15 @@ module Sidekiq
       class UniqueJobs
         HASH_KEY_EXPIRATION = 30 * 60
 
-        def initialize(redis)
-          @redis = redis
-        end
-
         def call(item, queue)
           payload_hash = Digest::MD5.hexdigest(MultiJson.encode(item))
-          return if already_scheduled?(payload_hash)
-
-          @redis.setex(payload_hash, HASH_KEY_EXPIRATION, 1)
+          Sidekiq.redis.with_connection do |redis|
+            return if redis.get(payload_hash)
+            redis.setex(payload_hash, HASH_KEY_EXPIRATION, 1)
+          end
 
           yield
         end
-
-        private
-
-        def already_scheduled?(payload_hash)
-          !!@redis.get(payload_hash)
-        end
       end
     end
   end

data/lib/sidekiq/middleware/server/failure_jobs.rb

@@ -0,0 +1,24 @@
+module Sidekiq
+  module Middleware
+    module Server
+      class FailureJobs
+        def call(*args)
+          yield
+        rescue => e
+          data = {
+            :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+            :payload => args[1],
+            :exception => e.class.to_s,
+            :error => e.to_s,
+            :backtrace => e.backtrace,
+            :worker => args[1]['class'],
+            :queue => args[2]
+          }
+
+          Sidekiq.redis.rpush(:failed, MultiJson.encode(data))
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/server/unique_jobs.rb

@@ -2,14 +2,10 @@ module Sidekiq
   module Middleware
     module Server
       class UniqueJobs
-        def initialize(redis)
-          @redis = redis
-        end
-
         def call(*args)
           yield
         ensure
-          @redis.del(Digest::MD5.hexdigest(MultiJson.encode(args[1])))
+          Sidekiq.redis.del(Digest::MD5.hexdigest(MultiJson.encode(args[1])))
         end
       end
     end

data/lib/sidekiq/processor.rb

@@ -1,10 +1,10 @@
 require 'celluloid'
-
 require 'sidekiq/util'
-require 'sidekiq/middleware/chain'
+
 require 'sidekiq/middleware/server/active_record'
 require 'sidekiq/middleware/server/airbrake'
 require 'sidekiq/middleware/server/unique_jobs'
+require 'sidekiq/middleware/server/failure_jobs'
 
 module Sidekiq
   class Processor
@@ -12,16 +12,14 @@ module Sidekiq
     include Celluloid
 
     def self.middleware
-      @middleware ||= begin
-        chain = Middleware::Chain.new
+      raise "Sidekiq::Processor.middleware is now Sidekiq.server_middleware"
+    end
 
-        # default middleware
-        chain.register do
-          use Middleware::Server::Airbrake
-          use Middleware::Server::UniqueJobs, Sidekiq::Manager.redis
-          use Middleware::Server::ActiveRecord
-        end
-        chain
+    def self.default_middleware
+      Middleware::Chain.new do |m|
+        m.add Middleware::Server::Airbrake
+        m.add Middleware::Server::UniqueJobs
+        m.add Middleware::Server::ActiveRecord
       end
     end
 
@@ -34,7 +32,7 @@ module Sidekiq
       klass  = constantize(msg['class'])
       worker = klass.new
       stats(worker, msg, queue) do
-        self.class.middleware.invoke(worker, msg, queue) do
+        Sidekiq.server_middleware.invoke(worker, msg, queue) do
           worker.perform(*msg['args'])
         end
       end
@@ -47,7 +45,7 @@ module Sidekiq
     end
 
     def to_s
-      @str ||= "#{hostname}:#{Process.pid}-#{Thread.current.object_id}:default"
+      @str ||= "#{hostname}:#{process_id}-#{Thread.current.object_id}:default"
     end
 
     private

data/lib/sidekiq/redis_connection.rb

@@ -6,7 +6,7 @@ module Sidekiq
     def self.create(options={})
       url = options[:url] || ENV['REDISTOGO_URL'] || 'redis://localhost:6379/0'
       client = build_client(url, options[:namespace])
-      return ConnectionPool.new(:timeout => 1, :size => 25) { client } unless options[:use_pool] == false
+      return ConnectionPool.new(:timeout => 1, :size => (options[:size] || Sidekiq.options[:concurrency] || 25)) { client } unless options[:use_pool] == false
       client
     end
 

data/lib/sidekiq/util.rb

@@ -1,6 +1,9 @@
 require 'logger'
 
 module Sidekiq
+  ##
+  # This module is part of Sidekiq core and not intended for extensions.
+  #
   module Util
 
     def self.logger
@@ -39,7 +42,23 @@ module Sidekiq
     end
 
     def redis
-      Sidekiq::Manager.redis
+      Sidekiq.redis
+    end
+
+    def process_id
+      Sidekiq::Util.process_id
+    end
+
+    def self.process_id
+      @pid ||= begin
+        if Process.pid == 1
+          # Heroku does not expose pids.
+          require 'securerandom'
+          (SecureRandom.random_number * 4_000_000_000).floor.to_s(16)
+        else
+          Process.pid
+        end
+      end
     end
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/sidekiq/worker.rb

@@ -34,7 +34,7 @@ module Sidekiq
       end
 
       def queue(name)
-        Sidekiq::Client.queues[self.name] = name.to_s
+        Sidekiq::Client.queue_mappings[self.name] = name.to_s
       end
     end
   end

data/myapp/Gemfile

@@ -1,7 +1,5 @@
 source 'https://rubygems.org'
 
-gem 'jruby-openssl' if defined?(JRUBY_VERSION)
-
 gem 'rails', '3.2.1'
 gem 'sqlite3'
 gem 'sidekiq', :path => '..'

data/myapp/app/controllers/work_controller.rb

@@ -3,7 +3,31 @@ class WorkController < ApplicationController
     @count = rand(100)
     puts "Adding #{@count} jobs"
     @count.times do |x|
-      HardWorker.perform_async('bubba', x)
+      HardWorker.perform_async('bubba', 0.01, x)
     end
   end
+
+  def email
+    UserMailer.delay.greetings(Time.now)
+    render :nothing => true
+  end
+
+  def long
+    10.times do |x|
+      HardWorker.perform_async('bob', 10, x)
+    end
+    render :text => 'enqueued'
+  end
+
+  def delayed_post
+    p = Post.first
+    unless p
+      p = Post.create!(:title => "Title!", :body => 'Body!')
+      p2 = Post.create!(:title => "Other!", :body => 'Second Body!')
+    else
+      p2 = Post.second
+    end
+    p.delay.long_method(p2)
+    render :nothing => true
+  end
 end

data/myapp/app/mailers/user_mailer.rb

@@ -0,0 +1,9 @@
+class UserMailer < ActionMailer::Base
+  default from: "sidekiq@example.com"
+
+  def greetings(now)
+    @now = now
+    @hostname = `hostname`.strip
+    mail(:to => 'mperham@gmail.com', :subject => 'Ahoy Matey!')
+  end
+end

data/myapp/app/models/post.rb

@@ -1,2 +1,5 @@
 class Post < ActiveRecord::Base
+  def long_method(other_post)
+    puts "Running long method with #{self.id} and #{other_post.id}"
+  end
 end

data/myapp/app/views/user_mailer/greetings.html.erb

@@ -0,0 +1,3 @@
+<p>
+  Hi Mike, it's <%= @now %> and I'm at <%= @hostname %>.
+</p>

data/myapp/app/workers/hard_worker.rb

@@ -1,8 +1,8 @@
 class HardWorker
   include Sidekiq::Worker
 
-  def perform(name, count)
-    sleep 1
+  def perform(name, count, salt)
     print "#{Time.now}\n"
+    sleep count
   end
 end

data/myapp/config/initializers/sidekiq.rb

@@ -1 +1,6 @@
-Sidekiq::Client.redis = Sidekiq::RedisConnection.create(:namespace => 'resque')
+Sidekiq.configure_client do |config|
+  config.redis = Sidekiq::RedisConnection.create(:namespace => 'resque', :size => 2)
+end
+Sidekiq.configure_server do |config|
+  config.redis = Sidekiq::RedisConnection.create(:namespace => 'resque', :size => 25)
+end

data/myapp/config/routes.rb

@@ -3,4 +3,7 @@ require 'resque/server'
 Myapp::Application.routes.draw do
   mount Resque::Server.new, :at => '/resque'
   get "work" => "work#index"
+  get "work/email" => "work#email"
+  get "work/post" => "work#delayed_post"
+  get "work/long" => "work#long"
 end

data/sidekiq.gemspec

@@ -20,4 +20,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'multi_json'
   gem.add_development_dependency      'minitest'
   gem.add_development_dependency      'rake'
+  gem.add_development_dependency      'actionmailer'
+  gem.add_development_dependency      'activerecord'
 end

data/test/config.yml

@@ -0,0 +1,11 @@
+---
+:verbose:      false
+:namespace:    test_namespace
+:server:       127.0.0.1:1234
+:environment:  xzibit
+:require:      ./test/fake_env.rb
+:pidfile:      /tmp/sidekiq-config-test.pid
+:concurrency:  50
+:queues:
+  - [often, 2]
+  - [seldom, 1]

data/test/test_cli.rb

@@ -1,5 +1,6 @@
 require 'helper'
 require 'sidekiq/cli'
+require 'tempfile'
 
 class TestCli < MiniTest::Unit::TestCase
   describe 'with cli' do
@@ -21,17 +22,132 @@ class TestCli < MiniTest::Unit::TestCase
 
     it 'changes concurrency' do
       @cli.parse(['sidekiq', '-c', '60', '-r', './test/fake_env.rb'])
-      assert_equal 60, @cli.options[:processor_count]
+      assert_equal 60, Sidekiq.options[:concurrency]
     end
 
     it 'changes queues' do
       @cli.parse(['sidekiq', '-q', 'foo', '-r', './test/fake_env.rb'])
-      assert_equal ['foo'], @cli.options[:queues]
+      assert_equal ['foo'], Sidekiq.options[:queues]
     end
 
     it 'handles weights' do
       @cli.parse(['sidekiq', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
-      assert_equal %w(bar foo foo foo), @cli.options[:queues].sort
+      assert_equal %w(bar foo foo foo), Sidekiq.options[:queues].sort
+    end
+
+    describe 'with pidfile' do
+      before do
+        @tmp_file = Tempfile.new('sidekiq-test')
+        @tmp_path = @tmp_file.path
+        @tmp_file.close!
+
+        @cli.parse(['sidekiq', '-P', @tmp_path, '-r', './test/fake_env.rb'])
+      end
+
+      after do
+        File.unlink @tmp_path if File.exist? @tmp_path
+      end
+
+      it 'sets pidfile path' do
+        assert_equal @tmp_path, Sidekiq.options[:pidfile]
+      end
+
+      it 'writes pidfile' do
+        assert_equal File.read(@tmp_path).strip.to_i, Process.pid
+      end
+    end
+
+    describe 'with config file' do
+      before do
+        @cli.parse(['sidekiq', '-C', './test/config.yml'])
+      end
+
+      it 'takes a path' do
+        assert_equal './test/config.yml', Sidekiq.options[:config_file]
+      end
+
+      it 'sets verbose' do
+        refute Sidekiq.options[:verbose]
+      end
+
+      it 'sets namespace' do
+        assert_equal "test_namespace", Sidekiq.options[:namespace]
+      end
+
+      it 'sets require file' do
+        assert_equal './test/fake_env.rb', Sidekiq.options[:require]
+      end
+
+      it 'sets environment' do
+        assert_equal 'xzibit', Sidekiq.options[:environment]
+      end
+
+      it 'sets concurrency' do
+        assert_equal 50, Sidekiq.options[:concurrency]
+      end
+
+      it 'sets pid file' do
+        assert_equal '/tmp/sidekiq-config-test.pid', Sidekiq.options[:pidfile]
+      end
+
+      it 'sets queues' do
+        assert_equal 2, Sidekiq.options[:queues].select{ |q| q == 'often' }.length
+        assert_equal 1, Sidekiq.options[:queues].select{ |q| q == 'seldom' }.length
+      end
+    end
+
+    describe 'with config file and flags' do
+      before do
+        # We need an actual file here.
+        @tmp_lib_path = '/tmp/require-me.rb'
+        File.open(@tmp_lib_path, 'w') do |f|
+          f.puts "# do work"
+        end
+
+        @tmp_file = Tempfile.new('sidekiqr')
+        @tmp_path = @tmp_file.path
+        @tmp_file.close!
+
+        @cli.parse(['sidekiq',
+                    '-C', './test/config.yml',
+                    '-n', 'sweet_story_bro',
+                    '-e', 'snoop',
+                    '-c', '100',
+                    '-r', @tmp_lib_path,
+                    '-P', @tmp_path,
+                    '-q', 'often,7',
+                    '-q', 'seldom,3'])
+      end
+
+      after do
+        File.unlink @tmp_lib_path if File.exist? @tmp_lib_path
+        File.unlink @tmp_path if File.exist? @tmp_path
+      end
+
+      it 'uses concurrency flag' do
+        assert_equal 100, Sidekiq.options[:concurrency]
+      end
+
+      it 'uses namespace flag' do
+        assert_equal "sweet_story_bro", Sidekiq.options[:namespace]
+      end
+
+      it 'uses require file flag' do
+        assert_equal @tmp_lib_path, Sidekiq.options[:require]
+      end
+
+      it 'uses environment flag' do
+        assert_equal 'snoop', Sidekiq.options[:environment]
+      end
+
+      it 'uses pidfile flag' do
+        assert_equal @tmp_path, Sidekiq.options[:pidfile]
+      end
+
+      it 'sets queues' do
+        assert_equal 7, Sidekiq.options[:queues].select{ |q| q == 'often' }.length
+        assert_equal 3, Sidekiq.options[:queues].select{ |q| q == 'seldom' }.length
+      end
     end
 
     def new_cli