roundhouse-x 0.1.0 → 0.2.0

data/lib/roundhouse/processor.rb

@@ -53,9 +53,6 @@ module Roundhouse
             execute_job(worker, cloned(msg['args']))
           end
         end
-
-        # Put the queue back into rotation
-        Roundhouse.redis { |conn| Roundhouse::Monitor.push(conn, queue) }
       rescue Roundhouse::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
@@ -65,6 +62,8 @@ module Roundhouse
         handle_exception(ex, msg || { :message => msgstr })
         raise
       ensure
+        # Put the queue back into rotation
+        Roundhouse.redis { |conn| Roundhouse::Monitor.push(conn, queue) }
         work.acknowledge if ack
       end
 

data/lib/roundhouse/scheduled.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 require 'roundhouse'
 require 'roundhouse/util'
 require 'roundhouse/actor'

data/lib/roundhouse/script.rb

@@ -0,0 +1,49 @@
+require 'wolverine/script'
+require 'wolverine/lua_error'
+
+module Roundhouse
+  # Modifies the Shopify Wolverine library. The Roundhouse::Script will
+  # take a string and load it up to redis directly. No method-missing, no
+  # autoloading, no templating.
+  # Optionally pass a :name, so we can collect stats. Otherwise it will be the SHA1 hash
+  class Script < Wolverine::Script
+    def initialize content, options = {}
+      @content = content
+      @digest  = Digest::SHA1.hexdigest @content
+      @config  = options[:config] || fail('Must supply a Wolverine::Configuration object')
+      @name    = options[:name] || @digest
+    end
+
+    def statsd
+      "Wolverine.#{@name}"
+    end
+
+    def instrument eval_type
+      ret = nil
+      runtime = Benchmark.realtime { ret = yield }
+      @config.instrumentation.call @name, runtime, eval_type
+      ret
+    end
+
+    class Configuration
+      attr_reader :instrumentation
+
+      def initialize(&blk)
+        @instrumentation = blk || proc { |script_name, runtime, eval_type| Roundhouse.logger.info "REDIS LUA [#{eval_type}]: #{script_name} #{runtime}" }
+      end
+    end
+  end
+end
+
+# Mock the Wolverine class
+class Wolverine
+  class << self
+    # If we added Roundhouse into a Rails app and we fully configure StatsD and
+    # Wolverine, then we don't want this proxy to interfere
+    unless defined?(statsd_enabled?)
+      def statsd_enabled?
+        false
+      end
+    end
+  end
+end

data/lib/roundhouse/version.rb

@@ -1,3 +1,3 @@
 module Roundhouse
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/roundhouse/web.rb

@@ -127,7 +127,6 @@ module Roundhouse
       redirect_with_query("#{root_path}morgue")
     end
 
-
     get '/retries' do
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
@@ -218,15 +217,19 @@ module Roundhouse
       content_type :json
       Roundhouse.dump_json(
         roundhouse: {
-          processed:       roundhouse_stats.processed,
-          failed:          roundhouse_stats.failed,
-          busy:            roundhouse_stats.workers_size,
-          processes:       roundhouse_stats.processes_size,
-          enqueued:        roundhouse_stats.enqueued,
-          scheduled:       roundhouse_stats.scheduled_size,
-          retries:         roundhouse_stats.retry_size,
-          dead:            roundhouse_stats.dead_size,
-          default_latency: roundhouse_stats.default_queue_latency
+          processed:            roundhouse_stats.processed,
+          failed:               roundhouse_stats.failed,
+          busy:                 roundhouse_stats.workers_size,
+          processes:            roundhouse_stats.processes_size,
+          enqueued:             roundhouse_stats.enqueued,
+          scheduled:            roundhouse_stats.scheduled_size,
+          retries:              roundhouse_stats.retry_size,
+          dead:                 roundhouse_stats.dead_size,
+          in_rotation:          roundhouse_stats.in_rotation,
+          num_queues:           roundhouse_stats.num_queues,
+          num_empty_queues:     roundhouse_stats.num_empty_queues,
+          num_suspended_queues: roundhouse_stats.num_suspended_queues,
+          avg_queue_len:        roundhouse_stats.avg_queue_len,
         },
         redis: redis_stats
       )

data/roundhouse.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'connection_pool', '~> 2.2', '>= 2.2.0'
   gem.add_dependency                  'celluloid', '~> 0.17.0'
   gem.add_dependency                  'json', '~> 1.0'
+  gem.add_dependency                  'wolverine', '~> 0.3.3'
   gem.add_development_dependency      'sinatra', '~> 1.4', '>= 1.4.6'
   gem.add_development_dependency      'minitest', '~> 5.7', '>= 5.7.0'
   gem.add_development_dependency      'rake', '~> 10.0'

data/test/test_cli.rb

@@ -41,11 +41,6 @@ class TestCli < Roundhouse::Test
       assert_equal 60, Roundhouse.options[:concurrency]
     end
 
-    it 'changes queues' do
-      @cli.parse(['roundhouse', '-q', 'foo', '-r', './test/fake_env.rb'])
-      assert_equal ['foo'], Roundhouse.options[:queues]
-    end
-
     it 'accepts a process index' do
       @cli.parse(['roundhouse', '-i', '7', '-r', './test/fake_env.rb'])
       assert_equal 7, Roundhouse.options[:index]
@@ -56,41 +51,11 @@ class TestCli < Roundhouse::Test
       assert_equal 7, Roundhouse.options[:index]
     end
 
-    it 'sets strictly ordered queues if weights are not present' do
-      @cli.parse(['roundhouse', '-q', 'foo', '-q', 'bar', '-r', './test/fake_env.rb'])
-      assert_equal true, !!Roundhouse.options[:strict]
-    end
-
-    it 'does not set strictly ordered queues if weights are present' do
-      @cli.parse(['roundhouse', '-q', 'foo,3', '-r', './test/fake_env.rb'])
-      assert_equal false, !!Roundhouse.options[:strict]
-    end
-
-    it 'does not set strictly ordered queues if weights are present with multiple queues' do
-      @cli.parse(['roundhouse', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
-      assert_equal false, !!Roundhouse.options[:strict]
-    end
-
     it 'changes timeout' do
       @cli.parse(['roundhouse', '-t', '30', '-r', './test/fake_env.rb'])
       assert_equal 30, Roundhouse.options[:timeout]
     end
 
-    it 'handles multiple queues with weights' do
-      @cli.parse(['roundhouse', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
-      assert_equal %w(foo foo foo bar), Roundhouse.options[:queues]
-    end
-
-    it 'handles queues with multi-word names' do
-      @cli.parse(['roundhouse', '-q', 'queue_one', '-q', 'queue-two', '-r', './test/fake_env.rb'])
-      assert_equal %w(queue_one queue-two), Roundhouse.options[:queues]
-    end
-
-    it 'handles queues with dots in the name' do
-      @cli.parse(['roundhouse', '-q', 'foo.bar', '-r', './test/fake_env.rb'])
-      assert_equal ['foo.bar'], Roundhouse.options[:queues]
-    end
-
     it 'sets verbose' do
       old = Roundhouse.logger.level
       @cli.parse(['roundhouse', '-v', '-r', './test/fake_env.rb'])
@@ -195,11 +160,6 @@ class TestCli < Roundhouse::Test
       it 'sets logfile' do
         assert_equal '/tmp/roundhouse.log', Roundhouse.options[:logfile]
       end
-
-      it 'sets queues' do
-        assert_equal 2, Roundhouse.options[:queues].count { |q| q == 'very_often' }
-        assert_equal 1, Roundhouse.options[:queues].count { |q| q == 'seldom' }
-      end
     end
 
     describe 'with env based config file' do
@@ -234,11 +194,6 @@ class TestCli < Roundhouse::Test
       it 'sets logfile' do
         assert_equal '/tmp/roundhouse.log', Roundhouse.options[:logfile]
       end
-
-      it 'sets queues' do
-        assert_equal 2, Roundhouse.options[:queues].count { |q| q == 'very_often' }
-        assert_equal 1, Roundhouse.options[:queues].count { |q| q == 'seldom' }
-      end
     end
 
     describe 'with an empty config file' do
@@ -287,9 +242,7 @@ class TestCli < Roundhouse::Test
                     '-e', 'snoop',
                     '-c', '100',
                     '-r', @tmp_lib_path,
-                    '-P', @tmp_path,
-                    '-q', 'often,7',
-                    '-q', 'seldom,3'])
+                    '-P', @tmp_path])
       end
 
       after do
@@ -312,53 +265,6 @@ class TestCli < Roundhouse::Test
       it 'uses pidfile flag' do
         assert_equal @tmp_path, Roundhouse.options[:pidfile]
       end
-
-      it 'sets queues' do
-        assert_equal 7, Roundhouse.options[:queues].count { |q| q == 'often' }
-        assert_equal 3, Roundhouse.options[:queues].count { |q| q == 'seldom' }
-      end
-    end
-
-    describe 'Roundhouse::CLI#parse_queues' do
-      describe 'when weight is present' do
-        it 'concatenates queues by factor of weight and sets strict to false' do
-          opts = { strict: true }
-          @cli.__send__ :parse_queues, opts, [['often', 7], ['repeatedly', 3]]
-          @cli.__send__ :parse_queues, opts, [['once']]
-          assert_equal (%w[often] * 7 + %w[repeatedly] * 3 + %w[once]), opts[:queues]
-          assert !opts[:strict]
-        end
-      end
-
-      describe 'when weight is not present' do
-        it 'returns queues and sets strict' do
-          opts = { strict: true }
-          @cli.__send__ :parse_queues, opts, [['once'], ['one_time']]
-          @cli.__send__ :parse_queues, opts, [['einmal']]
-          assert_equal %w[once one_time einmal], opts[:queues]
-          assert opts[:strict]
-        end
-      end
-    end
-
-    describe 'Roundhouse::CLI#parse_queue' do
-      describe 'when weight is present' do
-        it 'concatenates queue to opts[:queues] weight number of times and sets strict to false' do
-          opts = { strict: true }
-          @cli.__send__ :parse_queue, opts, 'often', 7
-          assert_equal %w[often] * 7, opts[:queues]
-          assert !opts[:strict]
-        end
-      end
-
-      describe 'when weight is not present' do
-        it 'concatenates queue to opts[:queues] once and leaves strict true' do
-          opts = { strict: true }
-          @cli.__send__ :parse_queue, opts, 'once', nil
-          assert_equal %w[once], opts[:queues]
-          assert opts[:strict]
-        end
-      end
     end
   end
 

data/test/test_fetch.rb

@@ -18,7 +18,7 @@ class TestFetcher < Roundhouse::Test
       describe 'when nominal' do
         before do
           Roundhouse.redis do |conn|
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 100)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 100)
             conn.hset(Roundhouse::Monitor.status_bucket(100), 100, Roundhouse::Monitor::ACTIVE)
             conn.lpush("#{Roundhouse::Monitor::QUEUE}:100", 'msg')
           end
@@ -41,68 +41,48 @@ class TestFetcher < Roundhouse::Test
       describe 'with empty status' do
         before do
           Roundhouse.redis do |conn|
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 100)
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 200)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 100)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 200)
             conn.hset(Roundhouse::Monitor.status_bucket(100), 100, Roundhouse::Monitor::EMPTY)
             conn.hset(Roundhouse::Monitor.status_bucket(200), 200, Roundhouse::Monitor::ACTIVE)
             conn.lpush("#{Roundhouse::Monitor::QUEUE}:200", 'msg-200')
           end
         end
 
-        it 'skips queues with empty status' do
+        it 'should return nil' do
           fetch = Roundhouse::RoundRobinFetch.new
           uow = fetch.retrieve_work
-          refute_nil uow
-          assert_equal '200', uow.queue_id
-          assert_equal 'msg-200', uow.message
-          q = Roundhouse::Queue.new(200)
-          assert_equal 0, q.size
-          uow.requeue
-          assert_equal 1, q.size
-          assert_nil uow.acknowledge
+          assert_nil uow
         end
       end
 
       describe 'with active status and empty queue' do
         before do
           Roundhouse.redis do |conn|
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 100)
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 200)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 100)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 200)
             conn.hset(Roundhouse::Monitor.status_bucket(100), 100, Roundhouse::Monitor::ACTIVE)
             conn.hset(Roundhouse::Monitor.status_bucket(200), 200, Roundhouse::Monitor::ACTIVE)
             conn.lpush("#{Roundhouse::Monitor::QUEUE}:200", 'msg-200')
           end
         end
 
-        it 'skips queues with empty status' do
+        it 'should return nil' do
           empty_queue = Roundhouse::Queue.new(100)
           assert_equal 0, empty_queue.size
           assert_equal :active, empty_queue.status
 
           fetch = Roundhouse::RoundRobinFetch.new
           uow = fetch.retrieve_work
-          refute_nil uow
-          assert_equal '200', uow.queue_id
-          assert_equal 'msg-200', uow.message
-
-          # After fetching, should set the active status
-          # of an empty queue to empty
-          assert_equal :empty, empty_queue.status
-
-          # The filled queue should act as normal
-          q = Roundhouse::Queue.new(200)
-          assert_equal 0, q.size
-          uow.requeue
-          assert_equal 1, q.size
-          assert_nil uow.acknowledge
+          assert_nil uow
         end
       end
 
       describe 'with suspended status' do
         before do
           Roundhouse.redis do |conn|
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 100)
-            conn.lpush(Roundhouse::Monitor::SEMAPHORE, 200)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 100)
+            conn.lpush(Roundhouse::Monitor::TURNTABLE, 200)
             conn.hset(Roundhouse::Monitor.status_bucket(100), 100, Roundhouse::Monitor::SUSPENDED)
             conn.hset(Roundhouse::Monitor.status_bucket(200), 200, Roundhouse::Monitor::ACTIVE)
             conn.lpush("#{Roundhouse::Monitor::QUEUE}:100", 'msg-100')
@@ -110,22 +90,10 @@ class TestFetcher < Roundhouse::Test
           end
         end
 
-        it 'skips queues with suspended status' do
+        it 'should return nil' do
           fetch = Roundhouse::RoundRobinFetch.new
           uow = fetch.retrieve_work
-          refute_nil uow
-          assert_equal '200', uow.queue_id
-          assert_equal 'msg-200', uow.message
-
-          # Should not touch the work in suspended queue
-          suspended_q = Roundhouse::Queue.new(100)
-          assert_equal 1, suspended_q.size
-
-          q = Roundhouse::Queue.new(200)
-          assert_equal 0, q.size
-          uow.requeue
-          assert_equal 1, q.size
-          assert_nil uow.acknowledge
+          assert_nil uow
         end
       end
 

data/test/test_monitor.rb

@@ -15,73 +15,102 @@ class TestMonitor < Roundhouse::Test
         it 'should pop the next available job from the next available queue' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+            assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
+
             assert_equal '100', Roundhouse::Monitor.pop(conn)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
           end
         end
       end
 
       describe 'with queue with empty status' do
-        it 'should skip' do
+        it 'should return nil' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.activate(conn, '200')
             Roundhouse::Monitor.push(conn, '200')
             Roundhouse::Monitor.set_queue_is_empty(conn, '200')
             Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+            assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 200)
 
-            assert_equal '100', Roundhouse::Monitor.pop(conn)
+            assert_equal nil, Roundhouse::Monitor.pop(conn)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 200)
           end
         end
       end
 
       describe 'with empty queue with active status' do
-        it 'should not skip' do
+        it 'should return queue_id' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.activate(conn, '200')
             Roundhouse::Monitor.push(conn, '200')
             Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+            assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 200)
 
             assert_equal '200', Roundhouse::Monitor.pop(conn)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 200)
           end
         end
       end
 
       describe 'with queue with suspended status' do
-        it 'should skip' do
+        it 'should return nil' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.activate(conn, '200')
             Roundhouse::Monitor.push(conn, '200')
             Roundhouse::Monitor.suspend(conn, '200')
             Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+            assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 200)
 
-            assert_equal '100', Roundhouse::Monitor.pop(conn)
+            assert_equal nil, Roundhouse::Monitor.pop(conn)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 200)
           end
         end
       end
 
-      describe 'when no active queue is in semaphore' do
-        it 'should block' do
-          skip 'do not know how to test this, and this may not be the right design'
+      describe 'when nothing is in the turntable' do
+        it "should block for #{Roundhouse::Monitor::TURNTABLE_TIMEOUT} seconds and return nil" do
+          skip 'skipping slow tests' unless ENV['SLOW']
+          Roundhouse.redis do |conn|
+            assert_equal nil, Roundhouse::Monitor.pop(conn)
+          end
         end
       end
     end # #pop
 
     describe '#push' do
       describe 'with active queue' do
-        it 'should push into semaphore' do
+        it 'should push into turntable' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
 
-            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, -1, -1)
+            assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
+          end
+        end
+
+        describe 'when already in turntable' do
+          it 'should not push into turntable' do
+            Roundhouse.redis do |conn|
+              Roundhouse::Monitor.activate(conn, 100)
+              Roundhouse::Monitor.push(conn, 100)
+              assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+              assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
+
+              Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+              assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+              assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
+            end
           end
         end
       end
 
       describe 'with queue with empty status' do
-        it 'should not push into semaphore' do
+        it 'should not push into turntable' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.set_queue_is_empty(conn, '200')
 
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
           end
         end
       end
@@ -92,7 +121,8 @@ class TestMonitor < Roundhouse::Test
             Roundhouse::Monitor.suspend(conn, '200')
             Roundhouse::Monitor.push(conn, '200')
 
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
           end
         end
       end
@@ -104,7 +134,11 @@ class TestMonitor < Roundhouse::Test
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
             assert conn.lrange("#{Roundhouse::Monitor::QUEUE}:100", 0, 0).first
+
             assert Roundhouse::Monitor.pop_job conn, 100
+
+            # Should still be active
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
           end
         end
       end
@@ -112,8 +146,14 @@ class TestMonitor < Roundhouse::Test
       describe 'without jobs in queue' do
         it 'should return nil' do
           Roundhouse.redis do |conn|
+            Roundhouse::Monitor.activate(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
             refute conn.lrange("#{Roundhouse::Monitor::QUEUE}:100", 0, 0).first
+
             refute Roundhouse::Monitor.pop_job conn, 100
+
+            # Should be set to inactive
+            assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
           end
         end
       end
@@ -142,31 +182,55 @@ class TestMonitor < Roundhouse::Test
 
     describe '#maybe_add_to_rotation' do
       describe 'with active queue' do
-        it 'should not rotate queue into semaphore' do
+        it 'should not rotate queue into turntable' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.activate(conn, 100)
             assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, 0)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
 
             Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
             assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, 0)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
           end
         end
       end
 
       describe 'with empty queue' do
-        it 'should rotate queue into semaphore' do
+        it 'should rotate queue into turntable' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.set_queue_is_empty(conn, 100)
             assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
 
             Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
             assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, 0)
+            assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
           end
         end
+
+        describe 'when already in rotation' do
+          it 'should not rotate queue into turntable' do
+            Roundhouse.redis do |conn|
+              Roundhouse::Monitor.activate(conn, 100)
+              Roundhouse::Monitor.push(conn, 100)
+              Roundhouse::Monitor.set_queue_is_empty(conn, 100)
+
+              assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+              assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+              assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
+
+              Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
+              assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+              assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+              assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
+            end
+          end
+
+        end
       end
 
       describe 'with suspended queue' do
@@ -174,11 +238,13 @@ class TestMonitor < Roundhouse::Test
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.suspend(conn, 100)
             assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
 
             Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
             assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+            refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, 100)
           end
         end
       end
@@ -213,15 +279,33 @@ class TestMonitor < Roundhouse::Test
 
     describe '#resume' do
       describe 'with suspended queue' do
-        it 'should reactive queue and put it into rotation' do
+        it 'should reactivate queue and put it into rotation' do
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.suspend(conn, 100)
             assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
 
             Roundhouse::Monitor.resume(conn, 100)
             assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+          end
+        end
+
+        describe 'when queue is already in rotation' do
+          it 'should reactivate but not put into queue' do
+            Roundhouse.redis do |conn|
+              Roundhouse::Monitor.activate(conn, 100)
+              Roundhouse::Monitor.push(conn, 100) # Puts it into rotation
+
+              Roundhouse::Monitor.suspend(conn, 100)
+              assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+              assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+
+              Roundhouse::Monitor.resume(conn, 100)
+              assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+              # There should only be a single queue 100 in the turntable
+              assert_equal ['100'], conn.lrange(Roundhouse::Monitor::TURNTABLE, 0, -1)
+            end
           end
         end
       end
@@ -231,11 +315,11 @@ class TestMonitor < Roundhouse::Test
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.set_queue_is_empty(conn, 100)
             assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, -1, -1)
 
             Roundhouse::Monitor.resume(conn, 100)
             assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, -1, -1)
           end
         end
       end
@@ -245,14 +329,209 @@ class TestMonitor < Roundhouse::Test
           Roundhouse.redis do |conn|
             Roundhouse::Monitor.activate(conn, 100)
             assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, -1, -1)
 
             Roundhouse::Monitor.resume(conn, 100)
             assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
-            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::TURNTABLE, -1, -1)
           end
         end
       end
     end # #resume
+
+    describe '#rebuild_turntable!' do
+      describe 'with active-status queue' do
+        before do
+          Roundhouse.redis { |conn| Roundhouse::Monitor.activate(conn, 100) }
+        end
+
+        describe 'with someting in the queue' do
+          it 'should put queue into turntable' do
+            Roundhouse.redis { |conn| conn.lpush("#{Roundhouse::Monitor::QUEUE}:100", 'something') }
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should still be active
+              assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should be in turntable
+              assert_equal 1, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+              assert_equal '100', conn.rpop(Roundhouse::Monitor::TURNTABLE)
+            end
+          end
+
+        end
+
+        describe 'without anything in queue' do
+          it 'should set queue to empty and not put into turntable' do
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should set to empty
+              assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should not be in turntable
+              assert_equal 0, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+            end
+          end
+        end
+      end
+
+      describe 'with empty-status queue' do
+        before do
+          Roundhouse.redis { |conn| Roundhouse::Monitor.set_queue_is_empty(conn, 100) }
+        end
+
+        describe 'with someting in the queue' do
+          it 'should activate queue and put queue into turntable' do
+            Roundhouse.redis { |conn| conn.lpush("#{Roundhouse::Monitor::QUEUE}:100", 'something') }
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should be set to active
+              assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should be in turntable
+              assert_equal 1, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              assert conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+              assert_equal '100', conn.rpop(Roundhouse::Monitor::TURNTABLE)
+            end
+          end
+
+        end
+
+        describe 'without anything in queue' do
+          it 'should not put into turntable' do
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should still be empty
+              assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should not be in turntable
+              assert_equal 0, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+            end
+          end
+        end
+      end
+
+      describe 'with suspended-status queue' do
+        before do
+          Roundhouse.redis { |conn| Roundhouse::Monitor.suspend(conn, 100) }
+        end
+
+        describe 'with something in the queue' do
+          it 'should not put queue into turntable' do
+            Roundhouse.redis { |conn| conn.lpush("#{Roundhouse::Monitor::QUEUE}:100", 'something') }
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should still be suspended
+              assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should not be in turntable
+              assert_equal 0, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+            end
+          end
+
+        end
+
+        describe 'without anything in queue' do
+          it 'should not put into turntable' do
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should still be suspended
+              assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should not be in turntable
+              assert_equal 0, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+            end
+          end
+        end
+      end
+
+      describe 'with unknown-status queue' do
+        before do
+          Roundhouse.redis { |conn| Roundhouse::Monitor.send(:set_queue_status, conn, 100, -2000) }
+        end
+
+        describe 'with something in the queue' do
+          it 'should not put queue into turntable' do
+            Roundhouse.redis { |conn| conn.lpush("#{Roundhouse::Monitor::QUEUE}:100", 'something') }
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should be suspended
+              assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should not be in turntable
+              assert_equal 0, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+            end
+          end
+
+        end
+
+        describe 'without anything in queue' do
+          it 'should not put into turntable' do
+            Roundhouse::Monitor.rebuild_turntable!
+
+            Roundhouse.redis do |conn|
+              # Should be suspended
+              assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+              # Should not be in turntable
+              assert_equal 0, conn.llen(Roundhouse::Monitor::TURNTABLE)
+              refute conn.sismember(Roundhouse::Monitor::IN_ROTATION, '100')
+            end
+          end
+        end
+      end
+
+      it 'scaling test' do
+        skip 'slow test' unless ENV['SLOW']
+        require 'benchmark'
+
+        num_queues = 10_000
+        num_active = (num_queues * 0.7).to_i
+        num_empty  = (num_queues * 0.2).to_i
+        num_suspended = num_queues - num_active - num_empty
+        active_with_empty = (num_active * 0.25).to_i
+        active_with_active = num_active - active_with_empty
+        empty_with_something = (num_empty * 0.25).to_i
+        empty_with_empty = num_empty - empty_with_something
+
+        states = \
+          Array.new(active_with_empty)    { :active_with_empty } +
+          Array.new(active_with_active)   { :active_with_active } +
+          Array.new(empty_with_something) { :empty_with_something } +
+          Array.new(empty_with_empty)     { :empty_with_empty } +
+          Array.new(num_suspended)        { :suspended }
+
+        rand_msgs = -> { Array.new(rand(3) + 1) { 'msg' } }
+        puts "\nScaling Test:\nSetting up"
+        Roundhouse.redis do |conn|
+          conn.multi do
+            states.shuffle.each_with_index do |state, q_id|
+              case state
+              when :active_with_empty then
+                Roundhouse::Monitor.activate(conn, q_id)
+              when :empty_with_empty then
+                Roundhouse::Monitor.set_queue_is_empty(conn, q_id)
+              when :suspended then
+                Roundhouse::Monitor.set_queue_is_empty(conn, q_id)
+              when :active_with_active then
+                Roundhouse::Monitor.activate(conn, q_id)
+                conn.lpush("#{Roundhouse::Monitor::QUEUE}:#{q_id}", rand_msgs.())
+              when :empty_with_something then
+                Roundhouse::Monitor.set_queue_is_empty(conn, q_id)
+                conn.lpush("#{Roundhouse::Monitor::QUEUE}:#{q_id}", rand_msgs.())
+              end
+            end
+          end
+        end
+
+        res = Benchmark.measure { Roundhouse::Monitor.rebuild_turntable! }
+        puts res, ''
+      end
+    end # #rebuild_turntable!
   end
 end