roundhouse-x 0.1.0

data/test/test_cli.rb

@@ -0,0 +1,389 @@
+require_relative 'helper'
+require 'roundhouse/cli'
+require 'tempfile'
+
+class Roundhouse::CLI
+  def die(code)
+    @code = code
+  end
+
+  def valid?
+    !@code
+  end
+end
+
+class TestCli < Roundhouse::Test
+  describe 'CLI#parse' do
+
+    before do
+      @cli = Roundhouse::CLI.new
+      @opts = Roundhouse.options.dup
+    end
+
+    after do
+      Roundhouse.options = @opts
+    end
+
+    it 'does not require the specified Ruby code' do
+      @cli.parse(['roundhouse', '-r', './test/fake_env.rb'])
+      refute($LOADED_FEATURES.any? { |x| x =~ /fake_env/ })
+      assert @cli.valid?
+    end
+
+    it 'does not boot rails' do
+      refute defined?(::Rails::Application)
+      @cli.parse(['roundhouse', '-r', './myapp'])
+      refute defined?(::Rails::Application)
+    end
+
+    it 'changes concurrency' do
+      @cli.parse(['roundhouse', '-c', '60', '-r', './test/fake_env.rb'])
+      assert_equal 60, Roundhouse.options[:concurrency]
+    end
+
+    it 'changes queues' do
+      @cli.parse(['roundhouse', '-q', 'foo', '-r', './test/fake_env.rb'])
+      assert_equal ['foo'], Roundhouse.options[:queues]
+    end
+
+    it 'accepts a process index' do
+      @cli.parse(['roundhouse', '-i', '7', '-r', './test/fake_env.rb'])
+      assert_equal 7, Roundhouse.options[:index]
+    end
+
+    it 'accepts a stringy process index' do
+      @cli.parse(['roundhouse', '-i', 'worker.7', '-r', './test/fake_env.rb'])
+      assert_equal 7, Roundhouse.options[:index]
+    end
+
+    it 'sets strictly ordered queues if weights are not present' do
+      @cli.parse(['roundhouse', '-q', 'foo', '-q', 'bar', '-r', './test/fake_env.rb'])
+      assert_equal true, !!Roundhouse.options[:strict]
+    end
+
+    it 'does not set strictly ordered queues if weights are present' do
+      @cli.parse(['roundhouse', '-q', 'foo,3', '-r', './test/fake_env.rb'])
+      assert_equal false, !!Roundhouse.options[:strict]
+    end
+
+    it 'does not set strictly ordered queues if weights are present with multiple queues' do
+      @cli.parse(['roundhouse', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
+      assert_equal false, !!Roundhouse.options[:strict]
+    end
+
+    it 'changes timeout' do
+      @cli.parse(['roundhouse', '-t', '30', '-r', './test/fake_env.rb'])
+      assert_equal 30, Roundhouse.options[:timeout]
+    end
+
+    it 'handles multiple queues with weights' do
+      @cli.parse(['roundhouse', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
+      assert_equal %w(foo foo foo bar), Roundhouse.options[:queues]
+    end
+
+    it 'handles queues with multi-word names' do
+      @cli.parse(['roundhouse', '-q', 'queue_one', '-q', 'queue-two', '-r', './test/fake_env.rb'])
+      assert_equal %w(queue_one queue-two), Roundhouse.options[:queues]
+    end
+
+    it 'handles queues with dots in the name' do
+      @cli.parse(['roundhouse', '-q', 'foo.bar', '-r', './test/fake_env.rb'])
+      assert_equal ['foo.bar'], Roundhouse.options[:queues]
+    end
+
+    it 'sets verbose' do
+      old = Roundhouse.logger.level
+      @cli.parse(['roundhouse', '-v', '-r', './test/fake_env.rb'])
+      assert_equal Logger::DEBUG, Roundhouse.logger.level
+      # If we leave the logger at DEBUG it'll add a lot of noise to the test output
+      Roundhouse.options.delete(:verbose)
+      Roundhouse.logger.level = old
+    end
+
+    describe 'with logfile' do
+      before do
+        @old_logger = Roundhouse.logger
+        @tmp_log_path = '/tmp/roundhouse.log'
+      end
+
+      after do
+        Roundhouse.logger = @old_logger
+        Roundhouse.options.delete(:logfile)
+        File.unlink @tmp_log_path if File.exist?(@tmp_log_path)
+      end
+
+      it 'sets the logfile path' do
+        @cli.parse(['roundhouse', '-L', @tmp_log_path, '-r', './test/fake_env.rb'])
+
+        assert_equal @tmp_log_path, Roundhouse.options[:logfile]
+      end
+
+      it 'creates and writes to a logfile' do
+        @cli.parse(['roundhouse', '-L', @tmp_log_path, '-r', './test/fake_env.rb'])
+
+        Roundhouse.logger.info('test message')
+
+        assert_match(/test message/, File.read(@tmp_log_path), "didn't include the log message")
+      end
+
+      it 'appends messages to a logfile' do
+        File.open(@tmp_log_path, 'w') do |f|
+          f.puts 'already existent log message'
+        end
+
+        @cli.parse(['roundhouse', '-L', @tmp_log_path, '-r', './test/fake_env.rb'])
+
+        Roundhouse.logger.info('test message')
+
+        log_file_content = File.read(@tmp_log_path)
+        assert_match(/already existent/, log_file_content, "didn't include the old message")
+        assert_match(/test message/, log_file_content, "didn't include the new message")
+      end
+    end
+
+    describe 'with pidfile' do
+      before do
+        @tmp_file = Tempfile.new('roundhouse-test')
+        @tmp_path = @tmp_file.path
+        @tmp_file.close!
+
+        @cli.parse(['roundhouse', '-P', @tmp_path, '-r', './test/fake_env.rb'])
+      end
+
+      after do
+        File.unlink @tmp_path if File.exist? @tmp_path
+      end
+
+      it 'sets pidfile path' do
+        assert_equal @tmp_path, Roundhouse.options[:pidfile]
+      end
+
+      it 'writes pidfile' do
+        assert_equal File.read(@tmp_path).strip.to_i, Process.pid
+      end
+    end
+
+    describe 'with config file' do
+      before do
+        @cli.parse(['roundhouse', '-C', './test/config.yml'])
+      end
+
+      it 'takes a path' do
+        assert_equal './test/config.yml', Roundhouse.options[:config_file]
+      end
+
+      it 'sets verbose' do
+        refute Roundhouse.options[:verbose]
+      end
+
+      it 'sets require file' do
+        assert_equal './test/fake_env.rb', Roundhouse.options[:require]
+      end
+
+      it 'does not set environment' do
+        assert_equal nil, Roundhouse.options[:environment]
+      end
+
+      it 'sets concurrency' do
+        assert_equal 50, Roundhouse.options[:concurrency]
+      end
+
+      it 'sets pid file' do
+        assert_equal '/tmp/roundhouse-config-test.pid', Roundhouse.options[:pidfile]
+      end
+
+      it 'sets logfile' do
+        assert_equal '/tmp/roundhouse.log', Roundhouse.options[:logfile]
+      end
+
+      it 'sets queues' do
+        assert_equal 2, Roundhouse.options[:queues].count { |q| q == 'very_often' }
+        assert_equal 1, Roundhouse.options[:queues].count { |q| q == 'seldom' }
+      end
+    end
+
+    describe 'with env based config file' do
+      before do
+        @cli.parse(['roundhouse', '-e', 'staging', '-C', './test/env_based_config.yml'])
+      end
+
+      it 'takes a path' do
+        assert_equal './test/env_based_config.yml', Roundhouse.options[:config_file]
+      end
+
+      it 'sets verbose' do
+        refute Roundhouse.options[:verbose]
+      end
+
+      it 'sets require file' do
+        assert_equal './test/fake_env.rb', Roundhouse.options[:require]
+      end
+
+      it 'sets environment' do
+        assert_equal 'staging', Roundhouse.options[:environment]
+      end
+
+      it 'sets concurrency' do
+        assert_equal 5, Roundhouse.options[:concurrency]
+      end
+
+      it 'sets pid file' do
+        assert_equal '/tmp/roundhouse-config-test.pid', Roundhouse.options[:pidfile]
+      end
+
+      it 'sets logfile' do
+        assert_equal '/tmp/roundhouse.log', Roundhouse.options[:logfile]
+      end
+
+      it 'sets queues' do
+        assert_equal 2, Roundhouse.options[:queues].count { |q| q == 'very_often' }
+        assert_equal 1, Roundhouse.options[:queues].count { |q| q == 'seldom' }
+      end
+    end
+
+    describe 'with an empty config file' do
+      before do
+        @tmp_file = Tempfile.new('roundhouse-test')
+        @tmp_path = @tmp_file.path
+        @tmp_file.close!
+      end
+
+      after do
+        File.unlink @tmp_path if File.exist? @tmp_path
+      end
+
+      it 'takes a path' do
+        @cli.parse(['roundhouse', '-C', @tmp_path])
+        assert_equal @tmp_path, Roundhouse.options[:config_file]
+      end
+
+      it 'should have an identical options hash, except for config_file' do
+        @cli.parse(['roundhouse'])
+        old_options = Roundhouse.options.clone
+
+        @cli.parse(['roundhouse', '-C', @tmp_path])
+        new_options = Roundhouse.options.clone
+        refute_equal old_options, new_options
+
+        new_options.delete(:config_file)
+        assert_equal old_options, new_options
+      end
+    end
+
+    describe 'with config file and flags' do
+      before do
+        # We need an actual file here.
+        @tmp_lib_path = '/tmp/require-me.rb'
+        File.open(@tmp_lib_path, 'w') do |f|
+          f.puts "# do work"
+        end
+
+        @tmp_file = Tempfile.new('roundhouser')
+        @tmp_path = @tmp_file.path
+        @tmp_file.close!
+
+        @cli.parse(['roundhouse',
+                    '-C', './test/config.yml',
+                    '-e', 'snoop',
+                    '-c', '100',
+                    '-r', @tmp_lib_path,
+                    '-P', @tmp_path,
+                    '-q', 'often,7',
+                    '-q', 'seldom,3'])
+      end
+
+      after do
+        File.unlink @tmp_lib_path if File.exist? @tmp_lib_path
+        File.unlink @tmp_path if File.exist? @tmp_path
+      end
+
+      it 'uses concurrency flag' do
+        assert_equal 100, Roundhouse.options[:concurrency]
+      end
+
+      it 'uses require file flag' do
+        assert_equal @tmp_lib_path, Roundhouse.options[:require]
+      end
+
+      it 'uses environment flag' do
+        assert_equal 'snoop', Roundhouse.options[:environment]
+      end
+
+      it 'uses pidfile flag' do
+        assert_equal @tmp_path, Roundhouse.options[:pidfile]
+      end
+
+      it 'sets queues' do
+        assert_equal 7, Roundhouse.options[:queues].count { |q| q == 'often' }
+        assert_equal 3, Roundhouse.options[:queues].count { |q| q == 'seldom' }
+      end
+    end
+
+    describe 'Roundhouse::CLI#parse_queues' do
+      describe 'when weight is present' do
+        it 'concatenates queues by factor of weight and sets strict to false' do
+          opts = { strict: true }
+          @cli.__send__ :parse_queues, opts, [['often', 7], ['repeatedly', 3]]
+          @cli.__send__ :parse_queues, opts, [['once']]
+          assert_equal (%w[often] * 7 + %w[repeatedly] * 3 + %w[once]), opts[:queues]
+          assert !opts[:strict]
+        end
+      end
+
+      describe 'when weight is not present' do
+        it 'returns queues and sets strict' do
+          opts = { strict: true }
+          @cli.__send__ :parse_queues, opts, [['once'], ['one_time']]
+          @cli.__send__ :parse_queues, opts, [['einmal']]
+          assert_equal %w[once one_time einmal], opts[:queues]
+          assert opts[:strict]
+        end
+      end
+    end
+
+    describe 'Roundhouse::CLI#parse_queue' do
+      describe 'when weight is present' do
+        it 'concatenates queue to opts[:queues] weight number of times and sets strict to false' do
+          opts = { strict: true }
+          @cli.__send__ :parse_queue, opts, 'often', 7
+          assert_equal %w[often] * 7, opts[:queues]
+          assert !opts[:strict]
+        end
+      end
+
+      describe 'when weight is not present' do
+        it 'concatenates queue to opts[:queues] once and leaves strict true' do
+          opts = { strict: true }
+          @cli.__send__ :parse_queue, opts, 'once', nil
+          assert_equal %w[once], opts[:queues]
+          assert opts[:strict]
+        end
+      end
+    end
+  end
+
+  describe 'misc' do
+    it 'handles interrupts' do
+      cli = Roundhouse::CLI.new
+      assert_raises Interrupt do
+        cli.handle_signal('INT')
+      end
+      assert_raises Interrupt do
+        cli.handle_signal('TERM')
+      end
+      cli.handle_signal('USR2')
+      cli.handle_signal('TTIN')
+    end
+
+    it 'can fire events' do
+      count = 0
+      Roundhouse.options[:lifecycle_events][:startup] = [proc {
+        count += 1
+      }]
+      cli = Roundhouse::CLI.new
+      cli.fire_event(:startup)
+      assert_equal 1, count
+    end
+  end
+
+end

data/test/test_client.rb

@@ -0,0 +1,294 @@
+require_relative 'helper'
+require 'roundhouse/client'
+require 'roundhouse/worker'
+require 'roundhouse/api'
+
+require 'active_support/core_ext/numeric/time'
+
+class TestClient < Roundhouse::Test
+  describe 'with mock redis' do
+    before do
+      @redis = Minitest::Mock.new
+      def @redis.multi; [yield] * 2 if block_given?; end
+      def @redis.set(*); true; end
+      def @redis.sadd(*); true; end
+      def @redis.srem(*); true; end
+      def @redis.get(*); nil; end
+      def @redis.del(*); nil; end
+      def @redis.incrby(*); nil; end
+      def @redis.setex(*); true; end
+      def @redis.expire(*); true; end
+      def @redis.watch(*); true; end
+      def @redis.with_connection; yield self; end
+      def @redis.with; yield self; end
+      def @redis.exec; true; end
+      def @redis.hset(*); true; end
+      def @redis.hget(*); true; end
+      Roundhouse.instance_variable_set(:@redis, @redis)
+      Roundhouse::Client.instance_variable_set(:@default, nil)
+    end
+
+    after do
+      Roundhouse.redis = REDIS
+      Roundhouse::Client.instance_variable_set(:@default, nil)
+    end
+
+    it 'raises ArgumentError with invalid params' do
+      assert_raises ArgumentError do
+        Roundhouse::Client.push('foo', 1)
+      end
+
+      assert_raises ArgumentError do
+        Roundhouse::Client.push('foo', :class => 'Foo', :noargs => [1, 2])
+      end
+
+      assert_raises ArgumentError do
+        Roundhouse::Client.push('queue_id' => 'wrong', 'class' => MyWorker, 'noargs' => [1, 2])
+      end
+
+      assert_raises ArgumentError do
+        Roundhouse::Client.push('queue_id' => 1, 'class' => MyWorker, 'noargs' => [1, 2])
+      end
+
+      assert_raises ArgumentError do
+        Roundhouse::Client.push('queue_id' => 1, 'class' => 42, 'args' => [1, 2])
+      end
+
+      assert_raises ArgumentError do
+        Roundhouse::Client.push('queue_id' => 1, 'class' => MyWorker, 'args' => 1)
+      end
+    end
+
+    describe 'as instance' do
+      it 'can push' do
+        @redis.expect :lpush, 1, ['queue:1', Array]
+        client = Roundhouse::Client.new
+        Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+          jid = client.push('class' => 'Blah', 'args' => [1,2,3], 'queue_id' => 1)
+          assert_equal 24, jid.size
+        end
+      end
+
+      it 'allows local middleware modification' do
+        @redis.expect :lpush, 1, ['queue:1', Array]
+        $called = false
+        mware = Class.new { def call(worker_klass,msg,q,r); $called = true; msg;end }
+        client = Roundhouse::Client.new
+        client.middleware do |chain|
+          chain.add mware
+        end
+
+        Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+          client.push('class' => 'Blah', 'args' => [1,2,3], 'queue_id' => 1)
+        end
+
+        assert $called
+        assert client.middleware.exists?(mware)
+        refute Roundhouse.client_middleware.exists?(mware)
+      end
+    end
+
+    it 'pushes messages to redis' do
+      @redis.expect :lpush, 1, ['queue:1', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = Roundhouse::Client.push('queue_id' => 1, 'class' => MyWorker, 'args' => [1, 2])
+        assert pushed
+        assert_equal 24, pushed.size
+      end
+      @redis.verify
+    end
+
+    it 'pushes messages to redis using a String class' do
+      @redis.expect :lpush, 1, ['queue:1', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = Roundhouse::Client.push('queue_id' => 1, 'class' => 'MyWorker', 'args' => [1, 2])
+        assert pushed
+        assert_equal 24, pushed.size
+      end
+      @redis.verify
+    end
+
+    class MyWorker
+      include Roundhouse::Worker
+    end
+
+    it 'has default options' do
+      assert_equal Roundhouse.default_worker_options, MyWorker.get_roundhouse_options
+    end
+
+    it 'handles perform_async' do
+      @redis.expect :lpush, 1, ['queue:100', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = MyWorker.perform_async(100, 1, 2)
+        assert pushed
+      end
+      @redis.verify
+    end
+
+    it 'enqueues messages to redis' do
+      @redis.expect :lpush, 1, ['queue:100', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = Roundhouse::Client.enqueue_to(100, MyWorker, 1, 2)
+        assert pushed
+      end
+      @redis.verify
+    end
+
+    it 'enqueues messages to redis (delayed)' do
+      @redis.expect :zadd, 1, ['schedule', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = Roundhouse::Client.enqueue_to_in(100, 3.minutes, MyWorker, 1, 2)
+        assert pushed
+      end
+      @redis.verify
+    end
+
+    it 'enqueues messages to redis (delayed into past)' do
+      @redis.expect :lpush, 1, ['queue:100', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = Roundhouse::Client.enqueue_to_in(100, -3.minutes, MyWorker, 1, 2)
+        assert pushed
+      end
+      @redis.verify
+    end
+
+    class QueuedWorker
+      include Roundhouse::Worker
+      roundhouse_options :queue => :flimflam
+    end
+
+    it 'enqueues to the named queue' do
+      @redis.expect :lpush, 1, ['queue:100', Array]
+      Roundhouse::Monitor.stub :maybe_add_to_rotation, true do
+        pushed = QueuedWorker.perform_async(100, 1, 2)
+        assert pushed
+      end
+      @redis.verify
+    end
+
+    it 'retrieves queues' do
+      @redis.expect :smembers, ['0'], [Roundhouse::Monitor::BUCKETS]
+      @redis.expect :hkeys, ['100'], [Roundhouse::Monitor.status_bucket(100)]
+      assert_equal [100], Roundhouse::Queue.all.map(&:queue_id)
+    end
+  end
+
+  describe 'bulk' do
+    after do
+      Roundhouse::Queue.new(100).clear
+    end
+    it 'can push a large set of jobs at once' do
+      jids = Roundhouse::Client.push_bulk('class' => QueuedWorker, 'queue_id' => 100, 'args' => (1..1_000).to_a.map { |x| Array(x) })
+      assert_equal 1_000, jids.size
+    end
+
+    it 'can push a large set of jobs at once using a String class' do
+      jids = Roundhouse::Client.push_bulk('class' => 'QueuedWorker', 'queue_id' => 100, 'args' => (1..1_000).to_a.map { |x| Array(x) })
+      assert_equal 1_000, jids.size
+    end
+    it 'returns the jids for the jobs' do
+      Roundhouse::Client.push_bulk('class' => 'QueuedWorker', 'queue_id' => 100, 'args' => (1..2).to_a.map { |x| Array(x) }).each do |jid|
+        assert_match(/[0-9a-f]{12}/, jid)
+      end
+    end
+  end
+
+  class BaseWorker
+    include Roundhouse::Worker
+    roundhouse_options 'retry' => 'base'
+  end
+  class AWorker < BaseWorker
+  end
+  class BWorker < BaseWorker
+    roundhouse_options 'retry' => 'b'
+  end
+  class CWorker < BaseWorker
+    roundhouse_options 'retry' => 2
+  end
+
+  describe 'client middleware' do
+
+    class Stopper
+      def call(worker_class, message, queue, r)
+        raise ArgumentError unless r
+        yield if message['args'].first.odd?
+      end
+    end
+
+    it 'can stop some of the jobs from pushing' do
+      Roundhouse.client_middleware.add Stopper
+      begin
+        assert_equal nil, Roundhouse::Client.push('queue_id' => 100, 'class' => MyWorker, 'args' => [0])
+        assert_match(/[0-9a-f]{12}/, Roundhouse::Client.push('queue_id' => 100, 'class' => MyWorker, 'args' => [1]))
+        Roundhouse::Client.push_bulk('queue_id' => 100, 'class' => MyWorker, 'args' => [[0], [1]]).each do |jid|
+          assert_match(/[0-9a-f]{12}/, jid)
+        end
+      ensure
+        Roundhouse.client_middleware.remove Stopper
+      end
+    end
+  end
+
+  describe 'inheritance' do
+    it 'inherits roundhouse options' do
+      assert_equal 'base', AWorker.get_roundhouse_options['retry']
+      assert_equal 'b', BWorker.get_roundhouse_options['retry']
+    end
+  end
+
+  describe 'item normalization' do
+    it 'defaults retry to true' do
+      assert_equal true, Roundhouse::Client.new.__send__(:normalize_item, 'queue_id' => 100, 'class' => QueuedWorker, 'args' => [])['retry']
+    end
+
+    it "does not normalize numeric retry's" do
+      assert_equal 2, Roundhouse::Client.new.__send__(:normalize_item, 'queue_id' => 100, 'class' => CWorker, 'args' => [])['retry']
+    end
+  end
+
+  describe 'sharding' do
+    class DWorker < BaseWorker
+    end
+
+    it 'allows roundhouse_options to point to different Redi' do
+      skip 'sharding not needed right now'
+
+      conn = MiniTest::Mock.new
+      conn.expect(:multi, [0, 1])
+      DWorker.roundhouse_options('pool' => ConnectionPool.new(size: 1) { conn })
+      DWorker.perform_async(1,2,3)
+      conn.verify
+    end
+
+    it 'allows #via to point to different Redi' do
+      skip '#via not needed right now'
+
+      conn = MiniTest::Mock.new
+      conn.expect(:multi, [0, 1])
+      default = Roundhouse::Client.new.redis_pool
+      sharded_pool = ConnectionPool.new(size: 1) { conn }
+      Roundhouse::Client.via(sharded_pool) do
+        CWorker.perform_async(100, 1,2,3)
+        assert_equal sharded_pool, Roundhouse::Client.new.redis_pool
+        assert_raises RuntimeError do
+          Roundhouse::Client.via(default) do
+            # nothing
+          end
+        end
+      end
+      assert_equal default, Roundhouse::Client.new.redis_pool
+      conn.verify
+    end
+
+    it 'allows Resque helpers to point to different Redi' do
+      skip 'sharding not needed right now'
+
+      conn = MiniTest::Mock.new
+      conn.expect(:multi, []) { |*args, &block| block.call }
+      conn.expect(:zadd, 1, [String, Array])
+      DWorker.roundhouse_options('pool' => ConnectionPool.new(size: 1) { conn })
+      Roundhouse::Client.enqueue_to_in(100, 10, DWorker, 3)
+      conn.verify
+    end
+  end
+end