staugaard-cloudmaster 0.1.1

data/test/instance-pool-test.rb

@@ -0,0 +1,359 @@
+$:.unshift(File.join(ENV['AWS_HOME'], "app"))
+require 'test/unit'
+require 'MockAWS/clock'
+require 'logger_factory'
+require 'configuration'
+require 'pool_configuration'
+require 'aws_context'
+require 'pp'
+
+# Test the InstancePool class.
+class InstancePoolTests < Test::Unit::TestCase
+  def setup
+    LoggerFactory.setup("/tmp/test.out")
+    logf = LoggerFactory.create(:file)
+    Clock.set(0)
+    config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
+    Cloudmaster::Configuration.setup_config_files(config_files)
+    tc = Cloudmaster::Configuration.new([''], [:primes])
+    @cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
+    reporter = Cloudmaster::Reporter.setup(@cfg[:name], logf)
+    @cfg[:ami_id] = "ami-08856161"
+    @ec2 = AwsContext.setup(:mock, logf).ec2
+    @pool = Cloudmaster::InstancePool.new(reporter, @cfg)
+  end
+
+  def test_size
+    assert_equal(0, @pool.size)
+    @pool.add('iid-fake', '')
+    assert_equal(1, @pool.size)
+  end
+
+  def test_enumerator
+    count = 0
+    @pool.each {|i| count += 1}
+    assert_equal(0, count)
+    assert_equal(0, @pool.size)
+    @pool.add('iid-fake', '')
+    count = 0
+    @pool.each {|i| count += 1}
+    assert_equal(1, count)
+    assert_equal(1, @pool.size)
+  end
+
+  def test_delete
+    assert_equal(0, @pool.size)
+    inst = @pool.add('iid-fake', '')
+    assert_equal(1, @pool.size)
+    @pool.delete(inst)
+    assert_equal(0, @pool.size)
+  end
+
+  def test_find_by_id
+    assert_equal(nil, @pool.find_by_id('iid-fake'))
+    inst = @pool.add('iid-fake', '')
+    assert_equal(inst, @pool.find_by_id('iid-fake'))
+  end
+
+  def test_id_list
+    assert_equal([], @pool.id_list)
+    inst = @pool.add('iid-fake', '')
+    assert_equal(['iid-fake'], @pool.id_list)
+  end
+
+  def test_less_than_minimum
+    assert(!@pool.less_than_minimum?)
+    @cfg[:minimum_number_of_instances] = 1
+    assert(@pool.less_than_minimum?)
+  end
+
+  def test_greater_than_maximum
+    assert(!@pool.greater_than_maximum?)
+    inst = @pool.add('iid-fake1', '')
+    inst = @pool.add('iid-fake2', '')
+    inst = @pool.add('iid-fake3', '')
+    assert_equal(3, @pool.size)
+    assert(@pool.greater_than_maximum?)
+  end
+
+  def test_below_minimum_count
+    assert_equal(0, @pool.below_minimum_count)
+    @cfg[:minimum_number_of_instances] = 1
+    assert_equal(1, @pool.below_minimum_count)
+  end
+
+  def test_above_maximum_count
+    assert_equal(0, @pool.above_maximum_count)
+    inst = @pool.add('iid-fake1', '')
+    inst = @pool.add('iid-fake2', '')
+    inst = @pool.add('iid-fake3', '')
+    assert_equal(1, @pool.above_maximum_count)
+  end
+
+  def test_missing_public_dns_instances
+    assert_equal([], @pool.missing_public_dns_instances)
+    inst1 = @pool.add('iid-fake1', '')
+    assert_equal([inst1], @pool.missing_public_dns_instances)
+    inst2 = @pool.add('iid-fake2', 'present')
+    assert_equal([inst1], @pool.missing_public_dns_instances)
+  end
+
+  def test_missing_public_dns_ids
+    assert_equal([], @pool.missing_public_dns_ids)
+    inst1 = @pool.add('iid-fake1', '')
+    assert_equal(['iid-fake1'], @pool.missing_public_dns_ids)
+    inst2 = @pool.add('iid-fake2', 'present')
+    assert_equal(['iid-fake1'], @pool.missing_public_dns_ids)
+  end
+
+  def test_update_public_dns
+    assert_equal([], @pool.missing_public_dns_ids)
+    inst1 = @pool.add('iid-fake1', '')
+    assert_equal(['iid-fake1'], @pool.missing_public_dns_ids)
+    @pool.update_public_dns('iid-fake1', '')
+    assert_equal(['iid-fake1'], @pool.missing_public_dns_ids)
+    @pool.update_public_dns('iid-fake1', 'pub-dns-1')
+    assert_equal([], @pool.missing_public_dns_ids)
+  end
+
+  def test_hung_instances
+    inst1 = @pool.add('iid-fake1', '')
+    assert_equal([], @pool.hung_instances)
+    Clock.set(3600)
+    assert_equal([inst1], @pool.hung_instances)
+  end
+
+  def test_active_instances
+    inst1 = @pool.add('iid-fake1', '')
+    assert_equal([], @pool.active_instances)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake2', 
+      :state => 'active',
+      :load_estimate => 1,
+      :timestamp => Clock.at(5)}
+    @pool.update_status(msg)
+    assert_equal([], @pool.active_instances)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 1,
+      :timestamp => Clock.at(5)}
+    @pool.update_status(msg)
+    assert_equal([inst1], @pool.active_instances)
+  end
+
+  def test_shut_down_instances
+    assert_equal([], @pool.shut_down_instances)
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    assert_equal([], @pool.shut_down_instances)
+    @pool.shut_down([inst1])
+    assert_equal([inst1], @pool.shut_down_instances)
+  end
+
+  def test_active_idle_instances
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake2', 
+      :state => 'active',
+      :load_estimate => 0,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    # both are active, but only inst2 is idle
+    assert_equal([inst2], @pool.active_idle_instances)
+  end
+
+  def test_shut_down_idle_instances
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 1,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    @pool.shut_down([inst1, inst2])
+    # both shut down, but only one idle
+    assert_equal([inst2], @pool.shut_down_idle_instances)
+  end
+
+  def test_shut_down_timeout_instances
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    @pool.shut_down([inst1])
+    Clock.set(1800)
+    @pool.shut_down([inst2])
+    Clock.set(3595)
+    assert_equal([], @pool.shut_down_timeout_instances)
+    Clock.set(3605)
+    # both shut down, but only inst1 has timed out
+    assert_equal([inst1], @pool.shut_down_timeout_instances)
+  end
+
+  def test_state_change_time
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    Clock.set(5)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake2', 
+      :state => 'active',
+      :load_estimate => 0,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    assert_equal(5, @pool.state_change_time.time)
+  end
+
+  def test_excess_capacity
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake2', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    assert_equal(0.5, @pool.excess_capacity)
+  end
+
+  def test_total_load
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake2', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    assert_equal(1, @pool.total_load)
+  end
+
+  def test_active_set
+    inst1 = @pool.add('iid-fake1', 'dns-1')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    aset = YAML.load(@pool.active_set)
+   assert_equal(1, aset.size)
+   assert_equal('iid-fake1', aset[0][:id])
+   assert_equal(0.5, aset[0][:load_estimate])
+   assert_equal('dns-1', aset[0][:public_dns])
+  end
+
+  def test_sorted_by_lowest_load
+    inst1 = @pool.add('iid-fake1', '')
+    inst2 = @pool.add('iid-fake2', '')
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake1', 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    msg = { :type => 'status',
+      :instance_id => 'iid-fake2', 
+      :state => 'active',
+      :load_estimate => 0.4,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    assert_equal([inst2, inst1], @pool.sorted_by_lowest_load)
+  end
+  
+  def test_update_public_dns_all
+    id1 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    inst1 = @pool.add(id1, '')
+    id2 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    inst2 = @pool.add(id2, '')
+    msg = { :type => 'status',
+      :instance_id => id1, 
+      :state => 'active',
+      :load_estimate => 0.5,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    msg = { :type => 'status',
+      :instance_id => id2, 
+      :state => 'active',
+      :load_estimate => 0.4,
+      :timestamp => Clock.at(10)}
+    @pool.update_status(msg)
+    assert_equal([inst1, inst2], @pool.missing_public_dns_instances)
+    @ec2.set_public_dns(id1, "dns-1")
+    @pool.update_public_dns_all
+    assert_equal([inst2], @pool.missing_public_dns_instances)
+  end
+
+  def test_our_running_instances
+    assert_equal(0, @pool.our_running_instances.size)
+    id1 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    id2 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    assert_equal(2, @pool.our_running_instances.size)
+    @ec2.set_state(id1, "stopped")
+    assert_equal(1, @pool.our_running_instances.size)
+  end
+
+  def test_audit_existing_instances_discover
+    id1 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    id2 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    @pool.audit_existing_instances
+    assert_equal(2, @pool.size)   
+  end
+
+  def test_audit_existing_instances_delete
+    id1 = 'iid-fake1'
+    @pool.add(id1, 'dns-1')
+    id2 = @ec2.run_instances('ami-08856161')[:instances][0][:id]
+    @pool.add(id2, 'dns-2')
+    assert_equal(2, @pool.size)
+    @pool.audit_existing_instances
+    # one is known, te other is not
+    assert_equal(1, @pool.size)    
+  end
+
+  def test_start_n_instances
+    assert_equal(0, @pool.size)
+    @pool.start_n_instances(2)
+    assert_equal(2, @pool.size)
+  end
+
+  def test_stop_instances
+    @pool.start_n_instances(2)
+    assert_equal(2, @pool.size)
+    @pool.stop_instances([@pool.find{|i| true}])
+    assert_equal(1, @pool.size)
+  end
+
+  def test_shut_down
+    @pool.start_n_instances(2)
+    assert_equal(2, @pool.size)
+    inst = [@pool.find{|i| true}]
+    sd = @pool.shut_down(inst)
+    assert_equal(sd, inst)
+  end
+end

data/test/instance-test.rb

@@ -0,0 +1,98 @@
+$:.unshift(File.join(ENV['AWS_HOME'], "app"))
+require 'test/unit'
+require 'MockAWS/clock'
+require 'logger_factory'
+require 'configuration'
+require 'pool_configuration'
+require 'pp'
+
+# Test the Instance class.
+class InstanceTests < Test::Unit::TestCase
+  def setup
+    Clock.set(0)
+    config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
+    Cloudmaster::Configuration.setup_config_files(config_files)
+    tc = Cloudmaster::Configuration.new([''], [:primes])
+    cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
+    @inst = Cloudmaster::Instance.new('iid-fake', '', cfg)
+  end
+
+  def state_msg(t, st)
+    {:timestamp => Clock.at(t), :state => st}
+  end
+
+  def test_update_status
+    # verify more recent messages are stored, but old ones skipped    
+    @inst.update_status(state_msg(1, "active1"))
+    assert_equal(1, @inst.timestamp.time)
+    assert_equal(:active1, @inst.state)
+    @inst.update_status(state_msg(2, "active2"))
+    assert_equal(2, @inst.timestamp.time)
+    assert_equal(:active2, @inst.state)
+    @inst.update_status(state_msg(1, "active1"))
+    assert_equal(:active2, @inst.state)    
+  end
+
+  def test_time_time_since_status
+    assert_equal(0, @inst.time_since_status)
+    Clock.set(5)
+    assert_equal(5, @inst.time_since_status)
+    @inst.update_status(state_msg(10, "active"))
+    assert_equal(0, @inst.time_since_status)
+  end
+
+  def test_time_since_state_change
+    assert_equal(0, @inst.time_since_state_change)
+    Clock.set(5)
+    assert_equal(5, @inst.time_since_state_change)
+    @inst.update_status(state_msg(10, "active"))
+    assert_equal(0, @inst.time_since_state_change)
+  end
+
+  def test_time_since_startup
+    assert_equal(0, @inst.time_since_state_change)
+    Clock.set(5)
+    assert_equal(5, @inst.time_since_state_change)
+    Clock.set(10)
+    assert_equal(10, @inst.time_since_state_change)
+  end
+
+  def test_report
+    @inst.report
+    assert(true)
+  end
+
+  def test_minimum_lifetime_elapsed
+    assert(! @inst.minimum_lifetime_elapsed?)    
+    Clock.set(3600)
+    assert(@inst.minimum_lifetime_elapsed?)    
+  end
+
+  def test_minimum_lifetime_elapsed
+    assert(! @inst.watchdog_time_elapsed?)    
+    Clock.set(3600)
+    assert(@inst.watchdog_time_elapsed?)    
+  end
+
+  def test_shutdown
+    assert_equal(:startup, @inst.state)
+    @inst.update_status(state_msg(10, "active"))
+    assert_equal(:active, @inst.state)
+    Clock.set(5)
+    @inst.shutdown    
+    assert_equal(:shut_down, @inst.state)
+    assert_equal(5, @inst.state_change_time.time)
+  end
+
+  def test_activate
+    Clock.set(5)
+    @inst.shutdown
+    assert_equal(:shut_down, @inst.state)
+    @inst.activate
+    assert_equal(:active, @inst.state)
+    assert_equal(5, @inst.state_change_time.time)
+  end
+
+  def test_one
+  end
+end

data/test/job-policy-test.rb

@@ -0,0 +1,95 @@
+$:.unshift(File.join(ENV['AWS_HOME'], "app"))
+require 'test/unit'
+require 'MockAWS/clock'
+require 'logger_factory'
+require 'configuration'
+require 'pool_configuration'
+require 'policy_limit'
+require 'aws_context'
+require 'pp'
+
+# Test the JobPolicy class.
+# Make sure it adjusts the number of instances appropriately in all cases.
+class JobPolicyTests < Test::Unit::TestCase
+  def setup
+    LoggerFactory.setup("/tmp/test.out")
+    logf = LoggerFactory.create(:file)
+    Clock.set(0)
+    config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
+    Cloudmaster::Configuration.setup_config_files(config_files)
+    tc = Cloudmaster::Configuration.new([''], [:primes])
+    cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
+    reporter = Cloudmaster::Reporter.setup(cfg[:name], logf)
+    cfg[:ami_id] = "ami-08856161"
+    @cfg = cfg
+    AwsContext.setup(:mock, logf)
+    @pool = Cloudmaster::InstancePool.new(reporter, @cfg)
+    @nq = @cfg[:work_queue] = Cloudmaster::NamedQueue.new(@cfg[:work_queue_name])
+    @jp = Cloudmaster::PolicyFactory.create(@cfg[:policy], reporter, @cfg, @pool)
+    @lp = Cloudmaster::PolicyLimit.new(reporter, @cfg, @pool)
+    queue_name = 'primes-work'
+    @queue = AwsContext.instance.sqs.list_queues(queue_name).first
+    @message = 'this is a test'
+  end
+
+  def send_work(n)
+    n.times { AwsContext.instance.sqs.send_message(@queue, @message)}
+    @nq.read_queue_depth
+  end
+
+  def test_idle
+    assert_equal(0, @jp.adjust)
+  end
+
+  def test_start_more
+    send_work(1)
+    # with queue depth 1 ==> 1
+    assert_equal(1, @jp.adjust)
+    # with queue depth 2 ==> 1
+    send_work(1)
+    assert_equal(1, @jp.adjust)
+    # with queue depth 4 ==> 2
+    send_work(1)
+    send_work(1)
+    assert_equal(2, @jp.adjust)
+    # with queue depth 6 ==> 3
+    send_work(1)
+    send_work(1)
+    assert_equal(3, @jp.adjust)
+    # with queue depth 6 ==> 2 (start_limit == 2)
+    assert_equal(2, @lp.adjust(@jp.adjust))
+
+    # now start 1
+    @pool.start_n_instances(1)
+    # depth is 6
+    assert_equal(2, @jp.adjust)
+    # depth is 6, but can only start 1 (max limit == 2)
+    assert_equal(1, @lp.adjust(@jp.adjust))
+  end
+
+  def test_no_change
+    # there is work aut we are at max_limit
+    send_work(1)
+    @pool.start_n_instances(2)
+    assert_equal(0, @jp.adjust)
+  end
+
+  def test_stop_some
+    @pool.start_n_instances(2)
+    @pool.each {|i| i.activate}
+    # it stops all
+    assert_equal(-2, @jp.adjust)
+    # if min_instances == 1, then it only stops 1
+    @cfg[:minimum_number_of_instances] = 1
+    assert_equal(-2, @jp.adjust)
+    assert_equal(-1, @lp.adjust(@jp.adjust))
+    # put it back at 0
+    @cfg[:minimum_number_of_instances] = 0
+    @pool.start_n_instances(1)
+    @pool.each {|i| i.activate}
+    # it would stop all three, except for limit (stop_limit = 2)
+    assert_equal(-3, @jp.adjust)
+    assert_equal(-2, @lp.adjust(@jp.adjust))
+  end
+
+end

data/test/manual-policy-tests.rb

@@ -0,0 +1,63 @@
+$:.unshift(File.join(ENV['AWS_HOME'], "app"))
+require 'test/unit'
+require 'MockAWS/clock'
+require 'logger_factory'
+require 'configuration'
+require 'pool_configuration'
+require 'policy_limit'
+require 'aws_context'
+require 'pp'
+
+# Test the ManualPolicy class.
+# Make sure it adjusts the number of instances appropriately in all cases.
+class ManualPolicyTests < Test::Unit::TestCase
+  def setup
+    LoggerFactory.setup("/tmp/test.out")
+    logf = LoggerFactory.create(:file)
+    Clock.set(0)
+    config_files = ['aws-config.ini', 'default-config.ini', 'test-config.ini']
+    Cloudmaster::Configuration.setup_config_files(config_files)
+    tc = Cloudmaster::Configuration.new([], [:manual])
+    cfg = Cloudmaster::PoolConfiguration.new(tc.aws, tc.default, tc.pools[0])
+    reporter = Cloudmaster::Reporter.setup(cfg[:name], logf)
+    cfg[:ami_id] = "ami-08856161"
+    @cfg = cfg
+    AwsContext.setup(:mock, logf)
+    @pool = Cloudmaster::InstancePool.new(reporter, @cfg)
+    @mq = Cloudmaster::NamedQueue.new(@cfg[:manual_queue_name])
+    @mp = Cloudmaster::PolicyFactory.create(@cfg[:policy], reporter, @cfg, @pool)
+    @lp = Cloudmaster::PolicyLimit.new(reporter, @cfg, @pool)
+    queue_name = 'manual-manual'
+    @queue = AwsContext.instance.sqs.list_queues(queue_name).first
+  end
+
+  def send_work(n, adjust)
+    message = YAML.dump({ :adjust => adjust})
+    n.times { AwsContext.instance.sqs.send_message(@queue, message)}
+  end
+
+  def test_idle
+    assert_equal(0, @mp.adjust)
+  end
+
+  def test_adjust_up
+    send_work(1, 1)
+    assert_equal(1, @mp.adjust)
+    send_work(1, 2)
+    assert_equal(2, @mp.adjust)
+    send_work(2, 1)
+    assert_equal(2, @mp.adjust)
+    send_work(2, 1)
+    send_work(1, -1)
+    assert_equal(1, @mp.adjust)
+  end
+
+  def test_adjust_down
+    send_work(1, -1)
+    assert_equal(-1, @mp.adjust)
+    send_work(1, -2)
+    assert_equal(-2, @mp.adjust)
+    send_work(2, -1)
+    assert_equal(-2, @mp.adjust)
+  end
+end

data/test/named-queue-test.rb

@@ -0,0 +1,90 @@
+$:.unshift(File.join(ENV['AWS_HOME'], "app"))
+require 'test/unit'
+require 'MockAWS/clock'
+require 'logger_factory'
+require 'aws_context'
+require 'named_queue'
+require 'pp'
+
+# test the NamedQueue implementation.
+class NamedQueueTests < Test::Unit::TestCase
+  def setup
+    LoggerFactory.setup("/tmp/test.out")
+    logf = LoggerFactory.create(:file)
+    @sqs = AwsContext.setup(:mock, logf).sqs
+  end
+
+  def test_none
+    nq = Cloudmaster::NamedQueue.new('xxx')
+    assert(true)
+  end
+
+  def test_one
+    nq = Cloudmaster::NamedQueue.new('primes-work')
+    assert(true)
+  end
+
+  def test_many
+    assert_raise(RuntimeError) do
+      nq = Cloudmaster::NamedQueue.new('primes')
+    end
+  end
+
+  def test_read_message
+    queue_name = 'primes-work'
+    queue = @sqs.list_queues(queue_name).first
+    nq = Cloudmaster::NamedQueue.new(queue_name)
+    msgs = nq.read_messages(1)
+    assert_equal([], msgs)
+    message = 'this is a test'
+    @sqs.send_message(queue, message)
+    # we can read
+    msgs = nq.read_messages(1)
+    assert_equal(1, msgs.size)
+    assert_equal(message, msgs.first[:body])    
+    msgs = nq.read_messages(1)
+    assert_equal(1, msgs.size)
+    assert_equal(message, msgs.first[:body])    
+    nq.delete_message(msgs.first[:receipt_handle])
+    msgs = nq.read_messages(1)
+    assert_equal(0, msgs.size)
+  end
+
+  def test_delete_message
+    queue_name = 'primes-work'
+    queue = @sqs.list_queues(queue_name).first
+    nq = Cloudmaster::NamedQueue.new(queue_name)
+    message = 'this is a test'
+    @sqs.send_message(queue, message)
+    msgs = nq.read_messages(1)
+    assert_equal(1, msgs.size)
+    assert_equal(message, msgs.first[:body])    
+    msgs = nq.read_messages(1)
+    nq.delete_message(msgs.first[:receipt_handle])
+    msgs = nq.read_messages(1)
+    assert_equal(0, msgs.size)
+  end
+
+  def test_read_queue_depth
+    queue_name = 'primes-work'
+    queue = @sqs.list_queues(queue_name).first
+    nq = Cloudmaster::NamedQueue.new(queue_name)
+    message = 'this is a test'
+    @sqs.send_message(queue, message)
+    @sqs.send_message(queue, message)
+    @sqs.send_message(queue, message)
+    assert_equal(3, nq.read_queue_depth)
+  end
+
+ def test_empty_queue
+    queue_name = 'primes-work'
+    queue = @sqs.list_queues(queue_name).first
+    nq = Cloudmaster::NamedQueue.new(queue_name)
+    message = 'this is a test'
+    @sqs.send_message(queue, message)
+    @sqs.send_message(queue, message)
+    @sqs.send_message(queue, message)
+    nq.empty_queue
+    assert_equal(0, nq.read_queue_depth)
+  end
+end