roundhouse-x 0.1.0

data/test/test_retry.rb

@@ -0,0 +1,390 @@
+# encoding: utf-8
+require_relative 'helper'
+require 'roundhouse/scheduled'
+require 'roundhouse/middleware/server/retry_jobs'
+
+class TestRetry < Roundhouse::Test
+  describe 'middleware' do
+    before do
+      @redis = Minitest::Mock.new
+      # Ugh, this is terrible.
+      Roundhouse.instance_variable_set(:@redis, @redis)
+
+      def @redis.with; yield self; end
+      def @redis.multi; yield self; end
+    end
+
+    after do
+      Roundhouse.redis = REDIS
+    end
+
+    let(:worker) do
+      Class.new do
+        include ::Roundhouse::Worker
+      end
+    end
+
+    it 'allows disabling retry' do
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => false }
+      msg2 = msg.dup
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg2, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal msg, msg2
+    end
+
+    it 'allows a numeric retry' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => 2 }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      msg.delete('failed_at')
+      assert_equal({"class"=>"Bob", "args"=>[1, 2, "foo"], "retry"=>2, "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "retry_count"=>0}, msg)
+      @redis.verify
+    end
+
+    it 'allows 0 retry => no retry and dead queue' do
+      @redis.expect :zadd, 1, ['dead', Float, String]
+      @redis.expect :zremrangebyscore, 0, ['dead', String, Float]
+      @redis.expect :zremrangebyrank, 0, ['dead', Numeric, Numeric]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => 0 }
+      msg2 = msg.dup
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      msg.delete('failed_at')
+      expected = msg2.merge "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "retry_count"=>0
+      assert_equal expected, msg
+      @redis.verify # not called
+    end
+
+    it 'handles zany characters in error message, #1705' do
+      skip 'skipped! test requires ruby 2.1+' if RUBY_VERSION <= '2.1.0'
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => 2 }
+      msg2 = msg.dup
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg2, 'default') do
+          raise "kerblammo! #{195.chr}"
+        end
+      end
+      msg2.delete('failed_at')
+      assert_equal({"class"=>"Bob", "args"=>[1, 2, "foo"], "retry"=>2, "queue"=>"default", "error_message"=>"kerblammo! �", "error_class"=>"RuntimeError", "retry_count"=>0}, msg2)
+      @redis.verify
+    end
+
+
+    it 'allows a max_retries option in initializer' do
+      max_retries = 7
+      1.upto(max_retries) do
+        @redis.expect :zadd, 1, ['retry', String, String]
+      end
+      @redis.expect :zadd, 1, ['dead', Float, String]
+      @redis.expect :zremrangebyscore, 0, ['dead', String, Float]
+      @redis.expect :zremrangebyrank, 0, ['dead', Numeric, Numeric]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new({:max_retries => max_retries})
+      1.upto(max_retries + 1) do
+        assert_raises RuntimeError do
+          handler.call(worker, msg, 'default') do
+            raise "kerblammo!"
+          end
+        end
+      end
+      @redis.verify
+    end
+
+    it 'saves backtraces' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true, 'backtrace' => true }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      c = nil
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          c = caller(0); raise "kerblammo!"
+        end
+      end
+      assert msg["error_backtrace"]
+      assert_equal c[0], msg["error_backtrace"][0]
+      @redis.verify
+    end
+
+    it 'saves partial backtraces' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true, 'backtrace' => 3 }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      c = nil
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          c = caller(0)[0...3]; raise "kerblammo!"
+        end
+      end
+      assert msg["error_backtrace"]
+      assert_equal c, msg["error_backtrace"]
+      assert_equal 3, c.size
+      assert_equal 3, msg["error_backtrace"].size
+    end
+
+    it 'handles a new failed message' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', msg["queue"]
+      assert_equal 'kerblammo!', msg["error_message"]
+      assert_equal 'RuntimeError', msg["error_class"]
+      assert_equal 0, msg["retry_count"]
+      refute msg["error_backtrace"]
+      assert msg["failed_at"]
+      @redis.verify
+    end
+
+    it 'shuts down without retrying work-in-progress, which will resume' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises Roundhouse::Shutdown do
+        handler.call(worker, msg, 'default') do
+          raise Roundhouse::Shutdown
+        end
+      end
+      assert_raises(MockExpectationError, "zadd should not be called") do
+        @redis.verify
+      end
+    end
+
+    it 'shuts down cleanly when shutdown causes exception' do
+      skip('Not supported in Ruby < 2.1.0') if RUBY_VERSION < '2.1.0'
+
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises Roundhouse::Shutdown do
+        handler.call(worker, msg, 'default') do
+          begin
+            raise Roundhouse::Shutdown
+          rescue Interrupt
+            raise "kerblammo!"
+          end
+        end
+      end
+      assert_raises(MockExpectationError, "zadd should not be called") do
+        @redis.verify
+      end
+    end
+
+    it 'shuts down cleanly when shutdown causes chained exceptions' do
+      skip('Not supported in Ruby < 2.1.0') if RUBY_VERSION < '2.1.0'
+
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises Roundhouse::Shutdown do
+        handler.call(worker, msg, 'default') do
+          begin
+            raise Roundhouse::Shutdown
+          rescue Interrupt
+            begin
+              raise "kerblammo!"
+            rescue
+              raise "kablooie!"
+            end
+          end
+        end
+      end
+      assert_raises(MockExpectationError, "zadd should not be called") do
+        @redis.verify
+      end
+    end
+
+    it 'allows a retry queue' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true, 'retry_queue' => 'retry' }
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'retry', msg["queue"]
+      assert_equal 'kerblammo!', msg["error_message"]
+      assert_equal 'RuntimeError', msg["error_class"]
+      assert_equal 0, msg["retry_count"]
+      refute msg["error_backtrace"]
+      assert msg["failed_at"]
+      @redis.verify
+    end
+
+    it 'handles a recurring failed message' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      now = Time.now.to_f
+      msg = {"class"=>"Bob", "args"=>[1, 2, "foo"], 'retry' => true, "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry_count"=>10}
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', msg["queue"]
+      assert_equal 'kerblammo!', msg["error_message"]
+      assert_equal 'RuntimeError', msg["error_class"]
+      assert_equal 11, msg["retry_count"]
+      assert msg["failed_at"]
+      @redis.verify
+    end
+
+    it 'handles a recurring failed message before reaching user-specifed max' do
+      @redis.expect :zadd, 1, ['retry', String, String]
+      now = Time.now.to_f
+      msg = {"class"=>"Bob", "args"=>[1, 2, "foo"], 'retry' => 10, "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry_count"=>8}
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', msg["queue"]
+      assert_equal 'kerblammo!', msg["error_message"]
+      assert_equal 'RuntimeError', msg["error_class"]
+      assert_equal 9, msg["retry_count"]
+      assert msg["failed_at"]
+      @redis.verify
+    end
+
+    it 'throws away old messages after too many retries (using the default)' do
+      now = Time.now.to_f
+      msg = {"class"=>"Bob", "args"=>[1, 2, "foo"], "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry"=>true, "retry_count"=>25}
+      @redis.expect :zadd, 1, ['dead', Float, String]
+      @redis.expect :zremrangebyscore, 0, ['dead', String, Float]
+      @redis.expect :zremrangebyrank, 0, ['dead', Numeric, Numeric]
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      @redis.verify
+    end
+
+    it 'throws away old messages after too many retries (using user-specified max)' do
+      now = Time.now.to_f
+      msg = {"class"=>"Bob", "args"=>[1, 2, "foo"], "queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry"=>3, "retry_count"=>3}
+      @redis.expect :zadd, 1, ['dead', Float, String]
+      @redis.expect :zremrangebyscore, 0, ['dead', String, Float]
+      @redis.expect :zremrangebyrank, 0, ['dead', Numeric, Numeric]
+      handler = Roundhouse::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      @redis.verify
+    end
+
+    describe "custom retry delay" do
+      before do
+        @old_logger    = Roundhouse.logger
+        @tmp_log_path  = '/tmp/roundhouse-retries.log'
+        Roundhouse.logger = Logger.new(@tmp_log_path)
+      end
+
+      after do
+        Roundhouse.logger = @old_logger
+        Roundhouse.options.delete(:logfile)
+        File.unlink @tmp_log_path if File.exist?(@tmp_log_path)
+      end
+
+      let(:custom_worker) do
+        Class.new do
+          include ::Roundhouse::Worker
+
+          roundhouse_retry_in do |count|
+            count * 2
+          end
+        end
+      end
+
+      let(:error_worker) do
+        Class.new do
+          include ::Roundhouse::Worker
+
+          roundhouse_retry_in do |count|
+            count / 0
+          end
+        end
+      end
+
+      let(:handler) { Roundhouse::Middleware::Server::RetryJobs.new }
+
+      it "retries with a default delay" do
+        refute_equal 4, handler.__send__(:delay_for, worker, 2)
+      end
+
+      it "retries with a custom delay" do
+        assert_equal 4, handler.__send__(:delay_for, custom_worker, 2)
+      end
+
+      it "falls back to the default retry on exception" do
+        refute_equal 4, handler.__send__(:delay_for, error_worker, 2)
+        assert_match(/Failure scheduling retry using the defined `roundhouse_retry_in`/,
+                     File.read(@tmp_log_path), 'Log entry missing for roundhouse_retry_in')
+      end
+    end
+
+    describe 'handles errors withouth cause' do
+      before do
+        @error = nil
+        begin
+          raise ::StandardError, 'Error'
+        rescue ::StandardError => e
+          @error = e
+        end
+      end
+
+      it "does not recurse infinitely checking if it's a shutdown" do
+        assert(!Roundhouse::Middleware::Server::RetryJobs.new.send(
+          :exception_caused_by_shutdown?, @error))
+      end
+    end
+
+    describe 'handles errors with circular causes' do
+      before do
+        @error = nil
+        begin
+          begin
+            raise ::StandardError, 'Error 1'
+          rescue ::StandardError => e1
+            begin
+              raise ::StandardError, 'Error 2'
+            rescue ::StandardError
+              raise e1
+            end
+          end
+        rescue ::StandardError => e
+          @error = e
+        end
+      end
+
+      it "does not recurse infinitely checking if it's a shutdown" do
+        assert(!Roundhouse::Middleware::Server::RetryJobs.new.send(
+          :exception_caused_by_shutdown?, @error))
+      end
+    end
+  end
+
+end

data/test/test_roundhouse.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+require_relative 'helper'
+
+class TestRoundhouse < Roundhouse::Test
+  describe 'json processing' do
+    it 'loads json' do
+      assert_equal ({"foo" => "bar"}), Roundhouse.load_json("{\"foo\":\"bar\"}")
+    end
+
+    it 'dumps json' do
+      assert_equal "{\"foo\":\"bar\"}", Roundhouse.dump_json({ "foo" => "bar" })
+    end
+  end
+
+  describe "redis connection" do
+  	it "returns error without creating a connection if block is not given" do
+      mock = Minitest::Mock.new
+      mock.expect :create, nil #Roundhouse::RedisConnection, create
+  		assert_raises(ArgumentError) {
+  			Roundhouse.redis
+  		}
+      assert_raises(MockExpectationError, "create should not be called") do
+        mock.verify
+      end
+  	end
+  end
+
+  describe "❨╯°□°❩╯︵┻━┻" do
+    before { $stdout = StringIO.new }
+    after  { $stdout = STDOUT }
+
+    it "allows angry developers to express their emotional constitution and remedies it" do
+      Roundhouse.❨╯°□°❩╯︵┻━┻
+      assert_equal "Calm down, yo.\n", $stdout.string
+    end
+  end
+
+  describe 'lifecycle events' do
+    it 'handles invalid input' do
+      Roundhouse.options[:lifecycle_events][:startup].clear
+
+      e = assert_raises ArgumentError do
+        Roundhouse.on(:startp)
+      end
+      assert_match(/Invalid event name/, e.message)
+      e = assert_raises ArgumentError do
+        Roundhouse.on('startup')
+      end
+      assert_match(/Symbols only/, e.message)
+      Roundhouse.on(:startup) do
+        1 + 1
+      end
+
+      assert_equal 2, Roundhouse.options[:lifecycle_events][:startup].first.call
+    end
+  end
+
+  describe 'default_worker_options' do
+    before do
+      @old_options = Roundhouse.default_worker_options
+    end
+    after  { Roundhouse.default_worker_options = @old_options }
+
+    it 'stringify keys' do
+      Roundhouse.default_worker_options = { queue: 'cat'}
+      assert_equal 'cat', Roundhouse.default_worker_options['queue']
+    end
+  end
+
+  describe 'error handling' do
+    it 'deals with user-specified error handlers which raise errors' do
+      output = capture_logging do
+        begin
+          Roundhouse.error_handlers << proc {|x, hash|
+            raise 'boom'
+          }
+          cli = Roundhouse::CLI.new
+          cli.handle_exception(RuntimeError.new("hello"))
+        ensure
+          Roundhouse.error_handlers.pop
+        end
+      end
+      assert_includes output, "boom"
+      assert_includes output, "ERROR"
+    end
+  end
+end

data/test/test_scheduled.rb

@@ -0,0 +1,120 @@
+require_relative 'helper'
+require 'roundhouse/scheduled'
+
+class TestScheduled < Roundhouse::Test
+  class ScheduledWorker
+    include Roundhouse::Worker
+    def perform(x)
+    end
+  end
+
+  describe 'poller' do
+    before do
+      Roundhouse.redis = REDIS
+      Roundhouse.redis do |conn|
+        conn.flushdb
+      end
+
+      @error_1  = { 'class' => ScheduledWorker.name, 'args' => [0], 'queue_id' => 100 }
+      @error_2  = { 'class' => ScheduledWorker.name, 'args' => [1], 'queue_id' => 200 }
+      @error_3  = { 'class' => ScheduledWorker.name, 'args' => [2], 'queue_id' => 300 }
+      @future_1 = { 'class' => ScheduledWorker.name, 'args' => [3], 'queue_id' => 400 }
+      @future_2 = { 'class' => ScheduledWorker.name, 'args' => [4], 'queue_id' => 500 }
+      @future_3 = { 'class' => ScheduledWorker.name, 'args' => [5], 'queue_id' => 600 }
+
+      @retry = Roundhouse::RetrySet.new
+      @scheduled = Roundhouse::ScheduledSet.new
+      @poller = Roundhouse::Scheduled::Poller.new
+    end
+
+    class Stopper
+      def call(worker_class, message, queue, r)
+        yield if message['args'].first.odd?
+      end
+    end
+
+    it 'executes client middleware' do
+      Roundhouse.client_middleware.add Stopper
+      begin
+        @retry.schedule (Time.now - 60).to_f, @error_1
+        @retry.schedule (Time.now - 60).to_f, @error_2
+        @scheduled.schedule (Time.now - 60).to_f, @future_2
+        @scheduled.schedule (Time.now - 60).to_f, @future_3
+
+        @poller.poll
+
+        Roundhouse.redis do |conn|
+          assert_equal 0, conn.llen("queue:100")
+          assert_equal 1, conn.llen("queue:200")
+          assert_equal 0, conn.llen("queue:500")
+          assert_equal 1, conn.llen("queue:600")
+        end
+      ensure
+        Roundhouse.client_middleware.remove Stopper
+      end
+    end
+
+    it 'should empty the retry and scheduled queues up to the current time' do
+      created_time  = Time.new(2013, 2, 3)
+      enqueued_time = Time.new(2013, 2, 4)
+
+      Time.stub(:now, created_time) do
+        @retry.schedule (enqueued_time - 60).to_f, @error_1.merge!('created_at' => created_time.to_f)
+        @retry.schedule (enqueued_time - 50).to_f, @error_2.merge!('created_at' => created_time.to_f)
+        @retry.schedule (enqueued_time + 60).to_f, @error_3.merge!('created_at' => created_time.to_f)
+        @scheduled.schedule (enqueued_time - 60).to_f, @future_1.merge!('created_at' => created_time.to_f)
+        @scheduled.schedule (enqueued_time - 50).to_f, @future_2.merge!('created_at' => created_time.to_f)
+        @scheduled.schedule (enqueued_time + 60).to_f, @future_3.merge!('created_at' => created_time.to_f)
+      end
+
+      Time.stub(:now, enqueued_time) do
+        @poller.poll
+
+        Roundhouse.redis do |conn|
+          %w(queue:100 queue:200 queue:400 queue:500).each do |queue_id|
+            assert_equal 1, conn.llen(queue_id)
+            job = Roundhouse.load_json(conn.lrange(queue_id, 0, -1)[0])
+            assert_equal enqueued_time.to_f, job['enqueued_at']
+            assert_equal created_time.to_f,  job['created_at']
+          end
+        end
+
+        assert_equal 1, @retry.size
+        assert_equal 1, @scheduled.size
+      end
+    end
+
+    def with_roundhouse_option(name, value)
+      _original, Roundhouse.options[name] = Roundhouse.options[name], value
+      begin
+        yield
+      ensure
+        Roundhouse.options[name] = _original
+      end
+    end
+
+    it 'generates random intervals that target a configured average' do
+      with_roundhouse_option(:poll_interval_average, 10) do
+        i = 500
+        intervals = i.times.map{ @poller.send(:random_poll_interval) }
+
+        assert intervals.all?{|x| x >= 5}
+        assert intervals.all?{|x| x <= 15}
+        assert_in_delta 10, intervals.reduce(&:+).to_f / i, 0.5
+      end
+    end
+
+    it 'calculates an average poll interval based on the number of known Roundhouse processes' do
+      with_roundhouse_option(:average_scheduled_poll_interval, 10) do
+        3.times do |i|
+          Roundhouse.redis do |conn|
+            conn.sadd("processes", "process-#{i}")
+            conn.hset("process-#{i}", "info", nil)
+          end
+        end
+
+        assert_equal 30, @poller.send(:scaled_poll_interval)
+      end
+    end
+  end
+end

data/test/test_scheduling.rb

@@ -0,0 +1,75 @@
+require_relative 'helper'
+require 'roundhouse/scheduled'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/integer/time'
+require 'active_support/core_ext/date'
+
+class TestScheduling < Roundhouse::Test
+  describe 'middleware' do
+    before do
+      Roundhouse::Client.instance_variable_set(:@default, nil)
+      @redis = Minitest::Mock.new
+      # Ugh, this is terrible.
+      Roundhouse.instance_variable_set(:@redis, @redis)
+      def @redis.multi; [yield] * 2 if block_given?; end
+      def @redis.with; yield self; end
+    end
+
+    after do
+      Roundhouse::Client.instance_variable_set(:@default, nil)
+      Roundhouse.instance_variable_set(:@redis, REDIS)
+    end
+
+    class ScheduledWorker
+      include Roundhouse::Worker
+      roundhouse_options :queue => :custom_queue
+      def perform(x)
+      end
+    end
+
+    it 'schedules a job via interval' do
+      @redis.expect :zadd, true, ['schedule', Array]
+      assert ScheduledWorker.perform_in(100, 600, 'mike')
+      @redis.verify
+    end
+
+    it 'schedules a job in one month' do
+      @redis.expect :zadd, true do |key, args|
+        assert_equal 'schedule', key
+        assert_in_delta 1.month.since.to_f, args[0][0].to_f, 1
+      end
+      assert ScheduledWorker.perform_in(100, 1.month, 'mike')
+      @redis.verify
+    end
+
+    it 'schedules a job via timestamp' do
+      @redis.expect :zadd, true, ['schedule', Array]
+      assert ScheduledWorker.perform_in(100, 5.days.from_now, 'mike')
+      @redis.verify
+    end
+
+    it 'schedules job right away on negative timestamp/interval' do
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        @redis.expect :lpush, true, ['queue:100', Array]
+        assert ScheduledWorker.perform_in(100, -300, 'mike')
+      end
+      @redis.verify
+    end
+
+    it 'schedules multiple jobs at once' do
+      @redis.expect :zadd, true, ['schedule', Array]
+      assert Roundhouse::Client.push_bulk('queue_id' => 100, 'class' => ScheduledWorker, 'args' => [['mike'], ['mike']], 'at' => 600)
+      @redis.verify
+    end
+
+    it 'removes the enqueued_at field when scheduling' do
+      @redis.expect :zadd, true do |key, args|
+        job = Roundhouse.load_json(args.first.last)
+        job.key?('created_at') && !job.key?('enqueued_at')
+      end
+      assert ScheduledWorker.perform_in(100, 1.month, 'mike')
+      @redis.verify
+    end
+  end
+
+end