roundhouse-x 0.1.0

data/test/test_monitor.rb

@@ -0,0 +1,258 @@
+require_relative 'helper'
+#require 'roundhouse/api'
+require 'roundhouse/monitor'
+
+require 'active_support/core_ext/numeric/time'
+
+class TestMonitor < Roundhouse::Test
+  describe 'Roundhouse::Monitor' do
+    before do
+      Roundhouse.redis(&:flushdb)
+    end
+
+    describe '#pop' do
+      describe 'when nominal' do
+        it 'should pop the next available job from the next available queue' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+            assert_equal '100', Roundhouse::Monitor.pop(conn)
+          end
+        end
+      end
+
+      describe 'with queue with empty status' do
+        it 'should skip' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.activate(conn, '200')
+            Roundhouse::Monitor.push(conn, '200')
+            Roundhouse::Monitor.set_queue_is_empty(conn, '200')
+            Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+
+            assert_equal '100', Roundhouse::Monitor.pop(conn)
+          end
+        end
+      end
+
+      describe 'with empty queue with active status' do
+        it 'should not skip' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.activate(conn, '200')
+            Roundhouse::Monitor.push(conn, '200')
+            Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+
+            assert_equal '200', Roundhouse::Monitor.pop(conn)
+          end
+        end
+      end
+
+      describe 'with queue with suspended status' do
+        it 'should skip' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.activate(conn, '200')
+            Roundhouse::Monitor.push(conn, '200')
+            Roundhouse::Monitor.suspend(conn, '200')
+            Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+
+            assert_equal '100', Roundhouse::Monitor.pop(conn)
+          end
+        end
+      end
+
+      describe 'when no active queue is in semaphore' do
+        it 'should block' do
+          skip 'do not know how to test this, and this may not be the right design'
+        end
+      end
+    end # #pop
+
+    describe '#push' do
+      describe 'with active queue' do
+        it 'should push into semaphore' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+
+            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+          end
+        end
+      end
+
+      describe 'with queue with empty status' do
+        it 'should not push into semaphore' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.set_queue_is_empty(conn, '200')
+
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+          end
+        end
+      end
+
+      describe 'with queue with suspended status' do
+        it 'should skip' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.suspend(conn, '200')
+            Roundhouse::Monitor.push(conn, '200')
+
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+          end
+        end
+      end
+    end # #push
+
+    describe '#pop_job' do
+      describe 'with jobs in queue' do
+        it 'should pop the next job and return data' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.push_job conn, [{'queue_id' => 100, 'class' => 'MyWorker', 'args' => [1]}]
+            assert conn.lrange("#{Roundhouse::Monitor::QUEUE}:100", 0, 0).first
+            assert Roundhouse::Monitor.pop_job conn, 100
+          end
+        end
+      end
+
+      describe 'without jobs in queue' do
+        it 'should return nil' do
+          Roundhouse.redis do |conn|
+            refute conn.lrange("#{Roundhouse::Monitor::QUEUE}:100", 0, 0).first
+            refute Roundhouse::Monitor.pop_job conn, 100
+          end
+        end
+      end
+    end
+
+    describe '#push_job' do
+      it 'should add bucket and push to queue' do
+        Roundhouse.redis do |conn|
+          bucket_num = Roundhouse::Monitor.bucket_num(2500)
+          refute conn.sismember(Roundhouse::Monitor::BUCKETS, bucket_num)
+          refute conn.lrange("#{Roundhouse::Monitor::QUEUE}:2500", 0, 0).first
+
+          Roundhouse::Monitor.push_job conn, [{'queue_id' => 2500, 'class' => 'MyWorker', 'args' => [1]}]
+
+          # We should see the bucket registered
+          assert conn.sismember(Roundhouse::Monitor::BUCKETS, bucket_num)
+          assert conn.lrange("#{Roundhouse::Monitor::QUEUE}:2500", 0, 0).first
+        end
+      end
+
+    end
+
+    describe '#requeue' do
+      it 'should requeue items'
+    end
+
+    describe '#maybe_add_to_rotation' do
+      describe 'with active queue' do
+        it 'should not rotate queue into semaphore' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.activate(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+
+            Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+          end
+        end
+      end
+
+      describe 'with empty queue' do
+        it 'should rotate queue into semaphore' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.set_queue_is_empty(conn, 100)
+            assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+
+            Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+          end
+        end
+      end
+
+      describe 'with suspended queue' do
+        it 'should not rotate queue into semaphore' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.suspend(conn, 100)
+            assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+
+            Roundhouse::Monitor.maybe_add_to_rotation(conn, 100)
+            assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, 0, 0)
+          end
+        end
+      end
+    end # maybe_add_to_rotation
+
+    describe '#set_queue_is_empty' do
+      it 'should set queue as empty' do
+        Roundhouse.redis do |conn|
+          Roundhouse::Monitor.set_queue_is_empty(conn, 100)
+          assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+        end
+      end
+    end # #set_queue_is_empty
+
+    describe '#activate' do
+      it 'should set queue as active' do
+        Roundhouse.redis do |conn|
+          Roundhouse::Monitor.activate(conn, 100)
+          assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+        end
+      end
+    end # #activate
+
+    describe '#suspend' do
+      it 'should set queue as suspended' do
+        Roundhouse.redis do |conn|
+          Roundhouse::Monitor.suspend(conn, 100)
+          assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+        end
+      end
+    end # #suspend
+
+    describe '#resume' do
+      describe 'with suspended queue' do
+        it 'should reactive queue and put it into rotation' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.suspend(conn, 100)
+            assert_equal Roundhouse::Monitor::SUSPENDED, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+
+            Roundhouse::Monitor.resume(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal ['100'], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+          end
+        end
+      end
+
+      describe 'with empty queue' do
+        it 'should not resume' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.set_queue_is_empty(conn, 100)
+            assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+
+            Roundhouse::Monitor.resume(conn, 100)
+            assert_equal Roundhouse::Monitor::EMPTY, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+          end
+        end
+      end
+
+      describe 'with active queue' do
+        it 'should not resume' do
+          Roundhouse.redis do |conn|
+            Roundhouse::Monitor.activate(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+
+            Roundhouse::Monitor.resume(conn, 100)
+            assert_equal Roundhouse::Monitor::ACTIVE, Roundhouse::Monitor.queue_status(conn, 100)
+            assert_equal [], conn.lrange(Roundhouse::Monitor::SEMAPHORE, -1, -1)
+          end
+        end
+      end
+    end # #resume
+  end
+end

data/test/test_processor.rb

@@ -0,0 +1,176 @@
+require_relative 'helper'
+require 'roundhouse/processor'
+require 'roundhouse/fetch'
+require 'roundhouse/cli'
+
+class TestProcessor < Roundhouse::Test
+  TestException = Class.new(StandardError)
+  TEST_EXCEPTION = TestException.new("kerboom!")
+
+  describe 'with mock setup' do
+    before do
+      $invokes = 0
+      @boss = Minitest::Mock.new
+      @processor = ::Roundhouse::Processor.new(@boss)
+      Celluloid.logger = nil
+      Roundhouse.redis = REDIS
+    end
+
+    class MockWorker
+      include Roundhouse::Worker
+      def perform(args)
+        raise TEST_EXCEPTION if args == 'boom'
+        args.pop if args.is_a? Array
+        $invokes += 1
+      end
+    end
+
+    def work(msg, full_queue_name='queue:100')
+      Roundhouse::RoundRobinFetch::UnitOfWork.new(full_queue_name, msg)
+    end
+
+    it 'processes as expected' do
+      # Queue needs to be active or it won't be put back into rotation
+      Roundhouse.redis { |conn| Roundhouse::Monitor.activate(conn, 100) }
+
+      msg = Roundhouse.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
+      actor = Minitest::Mock.new
+      actor.expect(:processor_done, nil, [@processor])
+      actor.expect(:real_thread, nil, [nil, Thread])
+      @boss.expect(:async, actor, [])
+      @boss.expect(:async, actor, [])
+      @processor.process(work(msg))
+
+      # After being finished, the q_id should be put back into the semaphore
+      queue_id = Roundhouse.redis { |conn| conn.lrange('semaphore', 0, 0).first }
+      assert_equal '100', queue_id
+
+      @boss.verify
+      assert_equal 1, $invokes
+    end
+
+    it 'executes a worker as expected' do
+      worker = Minitest::Mock.new
+      worker.expect(:perform, nil, [1, 2, 3])
+      @processor.execute_job(worker, [1, 2, 3])
+    end
+
+    it 'passes exceptions to ExceptionHandler' do
+      actor = Minitest::Mock.new
+      actor.expect(:real_thread, nil, [nil, Thread])
+      @boss.expect(:async, actor, [])
+      msg = Roundhouse.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+      begin
+        @processor.process(work(msg))
+        flunk "Expected #process to raise exception"
+      rescue TestException
+      end
+
+      assert_equal 0, $invokes
+    end
+
+    it 're-raises exceptions after handling' do
+      msg = Roundhouse.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+      re_raise = false
+      actor = Minitest::Mock.new
+      actor.expect(:real_thread, nil, [nil, Thread])
+      @boss.expect(:async, actor, [])
+
+      begin
+        @processor.process(work(msg))
+      rescue TestException
+        re_raise = true
+      end
+
+      assert re_raise, "does not re-raise exceptions after handling"
+    end
+
+    it 'does not modify original arguments' do
+      msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
+      msgstr = Roundhouse.dump_json(msg)
+      processor = ::Roundhouse::Processor.new(@boss)
+      actor = Minitest::Mock.new
+      actor.expect(:processor_done, nil, [processor])
+      actor.expect(:real_thread, nil, [nil, Thread])
+      @boss.expect(:async, actor, [])
+      @boss.expect(:async, actor, [])
+      processor.process(work(msgstr))
+      assert_equal [['myarg']], msg['args']
+    end
+
+    describe 'stats' do
+      before do
+        Roundhouse.redis {|c| c.flushdb }
+      end
+
+      def with_expire(time)
+        begin
+          old = Roundhouse::Processor::STATS_TIMEOUT
+          silence_warnings { Roundhouse::Processor.const_set(:STATS_TIMEOUT, time) }
+          yield
+        ensure
+          silence_warnings { Roundhouse::Processor.const_set(:STATS_TIMEOUT, old) }
+        end
+      end
+
+      describe 'when successful' do
+        let(:processed_today_key) { "stat:processed:#{Time.now.utc.to_date}" }
+
+        def successful_job
+          msg = Roundhouse.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
+          actor = Minitest::Mock.new
+          actor.expect(:real_thread, nil, [nil, Thread])
+          actor.expect(:processor_done, nil, [@processor])
+          @boss.expect(:async, actor, [])
+          @boss.expect(:async, actor, [])
+          @processor.process(work(msg))
+        end
+
+        it 'increments processed stat' do
+          successful_job
+          assert_equal 1, Roundhouse::Stats.new.processed
+        end
+
+        it 'expires processed stat' do
+          successful_job
+          assert_equal Roundhouse::Processor::STATS_TIMEOUT, Roundhouse.redis { |conn| conn.ttl(processed_today_key) }
+        end
+
+        it 'increments date processed stat' do
+          successful_job
+          assert_equal 1, Roundhouse.redis { |conn| conn.get(processed_today_key) }.to_i
+        end
+      end
+
+      describe 'when failed' do
+        let(:failed_today_key) { "stat:failed:#{Time.now.utc.to_date}" }
+
+        def failed_job
+          actor = Minitest::Mock.new
+          actor.expect(:real_thread, nil, [nil, Thread])
+          @boss.expect(:async, actor, [])
+          msg = Roundhouse.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+          begin
+            @processor.process(work(msg))
+          rescue TestException
+          end
+        end
+
+        it 'increments failed stat' do
+          failed_job
+          assert_equal 1, Roundhouse::Stats.new.failed
+        end
+
+        it 'increments date failed stat' do
+          failed_job
+          assert_equal 1, Roundhouse.redis { |conn| conn.get(failed_today_key) }.to_i
+        end
+
+        it 'expires failed stat' do
+          failed_job
+          assert_equal Roundhouse::Processor::STATS_TIMEOUT, Roundhouse.redis { |conn| conn.ttl(failed_today_key) }
+        end
+      end
+    end
+  end
+end

data/test/test_rails.rb

@@ -0,0 +1,23 @@
+require_relative 'helper'
+require 'roundhouse'
+require 'roundhouse/web_helpers'
+
+$HAS_AJ = true
+begin
+  require 'active_job'
+rescue LoadError
+  $HAS_AJ = false
+end
+
+class TestRails < Roundhouse::Test
+
+  describe 'ActiveJob' do
+    it 'does not allow Roundhouse::Worker in AJ::Base classes' do
+      ex = assert_raises ArgumentError do
+        c = Class.new(ActiveJob::Base)
+        c.send(:include, Roundhouse::Worker)
+      end
+      assert_includes ex.message, "cannot include"
+    end if $HAS_AJ
+  end
+end

data/test/test_redis_connection.rb

@@ -0,0 +1,127 @@
+require_relative 'helper'
+require 'roundhouse/redis_connection'
+
+class TestRedisConnection < Roundhouse::Test
+
+  describe ".create" do
+
+    it "creates a pooled redis connection" do
+      pool = Roundhouse::RedisConnection.create
+      assert_equal Redis, pool.checkout.class
+    end
+
+    describe "network_timeout" do
+      it "sets a custom network_timeout if specified" do
+        pool = Roundhouse::RedisConnection.create(:network_timeout => 8)
+        redis = pool.checkout
+
+        assert_equal 8, redis.client.timeout
+      end
+
+      it "uses the default network_timeout if none specified" do
+        pool = Roundhouse::RedisConnection.create
+        redis = pool.checkout
+
+        assert_equal 5, redis.client.timeout
+      end
+    end
+
+    describe "namespace" do
+      it "uses a given :namespace" do
+        pool = Roundhouse::RedisConnection.create(:namespace => "xxx")
+        assert_equal "xxx", pool.checkout.namespace
+      end
+
+      it "uses given :namespace over :namespace from Roundhouse.options" do
+        Roundhouse.options[:namespace] = "xxx"
+        pool = Roundhouse::RedisConnection.create(:namespace => "yyy")
+        assert_equal "yyy", pool.checkout.namespace
+      end
+    end
+
+    describe "socket path" do
+      it "uses a given :path" do
+        pool = Roundhouse::RedisConnection.create(:path => "/var/run/redis.sock")
+        assert_equal "unix", pool.checkout.client.scheme
+        assert_equal "redis:///var/run/redis.sock/0", pool.checkout.client.id
+      end
+
+      it "uses a given :path and :db" do
+        pool = Roundhouse::RedisConnection.create(:path => "/var/run/redis.sock", :db => 8)
+        assert_equal "unix", pool.checkout.client.scheme
+        assert_equal "redis:///var/run/redis.sock/8", pool.checkout.client.id
+      end
+    end
+
+    describe "pool_timeout" do
+      it "uses a given :timeout over the default of 1" do
+        pool = Roundhouse::RedisConnection.create(:pool_timeout => 5)
+
+        assert_equal 5, pool.instance_eval{ @timeout }
+      end
+
+      it "uses the default timeout of 1 if no override" do
+        pool = Roundhouse::RedisConnection.create
+
+        assert_equal 1, pool.instance_eval{ @timeout }
+      end
+    end
+  end
+
+  describe ".determine_redis_provider" do
+
+    before do
+      @old_env = ENV.to_hash
+    end
+
+    after do
+      ENV.update(@old_env)
+    end
+
+    def with_env_var(var, uri, skip_provider=false)
+      vars = ['REDISTOGO_URL', 'REDIS_PROVIDER', 'REDIS_URL'] - [var]
+      vars.each do |v|
+        next if skip_provider
+        ENV[v] = nil
+      end
+      ENV[var] = uri
+      assert_equal uri, Roundhouse::RedisConnection.__send__(:determine_redis_provider)
+      ENV[var] = nil
+    end
+
+    describe "with REDISTOGO_URL and a parallel REDIS_PROVIDER set" do
+      it "sets connection URI to the provider" do
+        uri = 'redis://roundhouse-redis-provider:6379/0'
+        provider = 'SIDEKIQ_REDIS_PROVIDER'
+
+        ENV['REDIS_PROVIDER'] = provider
+        ENV[provider] = uri
+        ENV['REDISTOGO_URL'] = 'redis://redis-to-go:6379/0'
+        with_env_var provider, uri, true
+
+        ENV[provider] = nil
+      end
+    end
+
+    describe "with REDIS_PROVIDER set" do
+      it "sets connection URI to the provider" do
+        uri = 'redis://roundhouse-redis-provider:6379/0'
+        provider = 'SIDEKIQ_REDIS_PROVIDER'
+
+        ENV['REDIS_PROVIDER'] = provider
+        ENV[provider] = uri
+
+        with_env_var provider, uri, true
+
+        ENV[provider] = nil
+      end
+    end
+
+    describe "with REDIS_URL set" do
+      it "sets connection URI to custom uri" do
+        with_env_var 'REDIS_URL', 'redis://redis-uri:6379/0'
+      end
+    end
+
+  end
+end