aboisvert_aws 3.0.0

data/right_aws.gemspec

@@ -0,0 +1,90 @@
+#--  -*- mode: ruby; encoding: utf-8 -*-
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'rubygems'
+require File.expand_path(File.join(File.dirname(__FILE__), "lib", "awsbase", "version"))
+
+Gem::Specification.new do |spec|
+  spec.name = 'aboisvert_aws'
+  spec.rubyforge_project = 'rightaws'
+  spec.version = RightAws::VERSION::STRING
+  spec.authors = ['RightScale, Inc.']
+  spec.email = 'support@rightscale.com'
+  spec.summary = 'Interface classes for the Amazon EC2, SQS, and S3 Web Services'
+  spec.rdoc_options = ['--main', 'README.txt', '--title', '']
+  spec.extra_rdoc_files = ['README.txt']
+  spec.required_ruby_version = '>= 1.8.7'
+  spec.require_path = 'lib'
+
+  spec.add_dependency('right_http_connection', '>= 1.2.5')
+  #spec.requirements << "uuidtools >= 1.0.7 if you want to use ActiveSdb"
+  spec.requirements << "libxml-ruby >= 0.5.2.0 is encouraged"
+
+  spec.add_development_dependency('rake')
+  spec.add_development_dependency('rcov')
+
+  spec.description = <<-EOF
+== DESCRIPTION:
+
+The RightScale AWS gems have been designed to provide a robust, fast, and secure interface to Amazon EC2, EBS, S3, SQS, SDB, and CloudFront.
+These gems have been used in production by RightScale since late 2006 and are being maintained to track enhancements made by Amazon.
+The RightScale AWS gems comprise:
+
+- RightAws::Ec2 -- interface to Amazon EC2 (Elastic Compute Cloud) and the
+  associated EBS (Elastic Block Store)
+- RightAws::S3 and RightAws::S3Interface -- interface to Amazon S3 (Simple Storage Service)
+- RightAws::Sqs and RightAws::SqsInterface -- interface to first-generation Amazon SQS (Simple Queue Service) (API version 2007-05-01)
+- RightAws::SqsGen2 and RightAws::SqsGen2Interface -- interface to second-generation Amazon SQS (Simple Queue Service) (API version 2008-01-01)
+- RightAws::SdbInterface and RightAws::ActiveSdb -- interface to Amazon SDB (SimpleDB)
+- RightAws::AcfInterface -- interface to Amazon CloudFront, a content distribution service
+
+== FEATURES:
+
+- Full programmmatic access to EC2, EBS, S3, SQS, SDB, and CloudFront.
+- Complete error handling: all operations check for errors and report complete
+  error information by raising an AwsError.
+- Persistent HTTP connections with robust network-level retry layer using
+  RightHttpConnection).  This includes socket timeouts and retries.
+- Robust HTTP-level retry layer.  Certain (user-adjustable) HTTP errors returned
+  by Amazon's services are classified as temporary errors.
+  These errors are automaticallly retried using exponentially increasing intervals.
+  The number of retries is user-configurable.
+- Fast REXML-based parsing of responses (as fast as a pure Ruby solution allows).
+- Uses libxml (if available) for faster response parsing.
+- Support for large S3 list operations.  Buckets and key subfolders containing
+  many (> 1000) keys are listed in entirety.  Operations based on list (like
+  bucket clear) work on arbitrary numbers of keys.
+- Support for streaming GETs from S3, and streaming PUTs to S3 if the data source is a file.
+- Support for single-threaded usage, multithreaded usage, as well as usage with multiple
+  AWS accounts.
+- Support for both first- and second-generation SQS (API versions 2007-05-01
+  and 2008-01-01).  These versions of SQS are not compatible.
+- Support for signature versions 0 and 1 on SQS, SDB, and EC2.
+- Interoperability with any cloud running Eucalyptus (http://eucalyptus.cs.ucsb.edu)
+- Test suite (requires AWS account to do "live" testing).
+EOF
+
+  candidates = Dir.glob('{lib,test}/**/*') + ['History.txt', 'Manifest.txt', 'README.txt', 'Rakefile', 'right_aws.gemspec']
+  spec.files = candidates.sort
+  spec.test_files = Dir.glob('test/**/*')
+end

data/test/README.mdown

@@ -0,0 +1,39 @@
+# Notes and tips for developers
+
+## Setting up credentials for testing
+
+Before you can run any tests, you need to set up credentials (API key and secret) that
+will be used when talking to AWS.  The credentials are loaded in `test/test_credentials.rb`
+and are expected to be found in `~/.rightscale/testcredentials.rb` and look like this:
+
+    TestCredentials.aws_access_key_id= 'AAAAAAAAAAAAAAAAAAAA'
+    TestCredentials.aws_secret_access_key= 'asdfasdfsadf'
+    TestCredentials.account_number= '???'
+
+If you prefer to store your secret key in the OS X keychain, you can do this:
+
+    def secret_access_key_from_keychain (key_id)
+      dump = `security -q find-generic-password -a "#{key_id}" -g 2>&1`
+      dump[/password: "(.*)"/, 1]
+    end
+
+    TestCredentials.aws_access_key_id= 'AAAAAAAAAAAAAAAAAAAA'
+    TestCredentials.aws_secret_access_key= secret_access_key_from_keychain(TestCredentials.aws_access_key_id)
+    TestCredentials.account_number= '???'
+
+## Running tests
+
+There is no test suite that runs all tests.  Each module is tested separately.  E.g.,
+to run the Load Balancer tests, run `rake testelb`.  Run `rake -T` for a full list.
+
+Some tests need to launch services on AWS to have something to test.  This means two things:
+
+1. Running all the tests will cost you money.
+2. You will need to shut down some services separately once you are done, or things
+    will keep running and cost you money.
+
+As an example, the ELB and Route 53 tests need a load balancer for testing.  Starting a load balancer
+for every test would make every test case cost as much as running the LB for one hour, so it makes
+more sense to leave it running until it's no longer needed.
+
+The ELB tests contain instructions for shutting down the load balancer.

data/test/acf/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../../lib/right_aws'

data/test/acf/test_right_acf.rb

@@ -0,0 +1,138 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestAcf < Test::Unit::TestCase
+
+  RIGHT_OBJECT_TEXT     = 'Right test message'
+  
+  STDOUT.sync = true
+
+  def setup
+    @acf= Rightscale::AcfInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key)
+    @s3 = Rightscale::S3.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key)
+    @bucket_name   = "right-acf-awesome-test-bucket-xxx1"
+    @bucket_domain = "#{@bucket_name}.s3.amazonaws.com"
+  end
+
+  def test_01_list_distributions_part1
+    distributions = nil
+    assert_nothing_raised(Rightscale::AwsError) do
+      distributions = @acf.list_distributions
+    end
+    assert distributions.is_a?(Array)
+  end
+
+  def test_02_try_to_create_for_bad_bucket
+    # a bucket does not exist
+    assert_raise(Rightscale::AwsError) do
+      @acf.create_distribution("right-cloudfront-awesome-test-bucket-not-exist", "Mustn't to be born", true)
+    end
+    # a bucket is not a domain naming complied guy
+    bucket_name = 'right_cloudfront_awesome_test_bucket_BAD_XXX'
+    @s3.bucket(bucket_name, :create)
+    assert_raise(Rightscale::AwsError) do
+      @acf.create_distribution(bucket_name, "Mustn't to be born", true)
+    end
+  end
+
+  def test_02_x_delete_bad_bucket
+    bucket_name = 'right_cloudfront_awesome_test_bucket_BAD_XXX'
+    @s3.bucket(bucket_name, false).delete
+  end
+
+  def test_03_create
+    comment = 'WooHoo!!!'
+    # create a test bucket
+    @s3.bucket(@bucket_name, :create)
+    # create a distribution
+    distribution = @acf.create_distribution(@bucket_domain, comment, true)
+    assert_equal comment, distribution[:comment]
+    assert       distribution[:cnames].size == 0
+    assert       distribution[:enabled]
+  end
+
+  def test_04_list_distributions_part2
+    distributions = @acf.list_distributions
+    assert distributions.size > 0
+  end
+
+  def get_test_distribution
+    @acf.list_distributions.select{ |d| d[:origin] == @bucket_domain }.first
+  end
+
+  def test_05_get_distribution
+    old = get_test_distribution
+    assert_nothing_raised do
+      @acf.get_distribution(old[:aws_id])
+    end
+  end
+
+  def test_06_get_and_set_config
+    config = nil
+    old = get_test_distribution
+    assert_nothing_raised do
+      config = @acf.get_distribution_config(old[:aws_id])
+    end
+    # change a config
+    config[:enabled] = false
+    config[:cnames] << 'xxx1.myawesomesite.com'
+    config[:cnames] << 'xxx2.myawesomesite.com'
+    # set config
+    set_config_result = nil
+    assert_nothing_raised do
+      set_config_result = @acf.set_distribution_config(old[:aws_id], config)
+    end
+    assert set_config_result
+    # reget the config and check
+    new_config = nil
+    assert_nothing_raised do
+      new_config = @acf.get_distribution_config(old[:aws_id])
+    end
+    assert           !new_config[:enabled]
+    assert_equal     new_config[:cnames].sort, ['xxx1.myawesomesite.com', 'xxx2.myawesomesite.com']
+    assert_not_equal config[:e_tag], new_config[:e_tag]
+
+    # try to update the old config again (must fail because ETAG has changed)
+    assert_raise(Rightscale::AwsError) do
+      @acf.set_distribution_config(old[:aws_id], config)
+    end
+  end
+
+  def test_08_delete_distribution
+    # we need ETAG so use get_distribution
+    distribution = @acf.get_distribution(get_test_distribution[:aws_id])
+    # try to delete a distribution
+    # should fail because
+    if distribution[:status] == 'InProgress'
+      # should fail because the distribution is not deployed yet
+      assert_raise(Rightscale::AwsError) do
+        @acf.delete_distribution(distribution[:aws_id], distribution[:e_tag])
+      end
+      # wait for a deployed state
+      print "waiting up to 5 min while the distribution is being deployed: "
+      100.times do
+        print '.'
+        distribution = @acf.get_distribution(distribution[:aws_id])
+        if distribution[:status] == 'Deployed'
+          print ' done'
+          break
+        end
+        sleep 3
+      end
+      puts
+    end
+
+    # only disabled and deployed distribution can be deleted
+    assert_equal 'Deployed', distribution[:status]
+    assert       !distribution[:enabled]
+
+    # delete the distribution
+    assert_nothing_raised do
+      @acf.delete_distribution(distribution[:aws_id], distribution[:e_tag])
+    end  
+  end
+
+  def test_09_drop_bucket
+    assert @s3.bucket(@bucket_name).delete
+  end
+
+end

data/test/awsbase/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../../lib/right_aws'

data/test/awsbase/test_right_awsbase.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestAwsbase < Test::Unit::TestCase
+
+  def setup
+  end
+  
+  def test_01_create_describe_key_pairs
+  end
+  
+end

data/test/ec2/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../../lib/right_aws'

data/test/ec2/test_right_ec2.rb

@@ -0,0 +1,107 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestEc2 < Test::Unit::TestCase
+
+    # Some of RightEc2 instance methods concerning instance launching and image registration
+    # are not tested here due to their potentially risk.
+  
+  def setup
+    @ec2   = Rightscale::Ec2.new(TestCredentials.aws_access_key_id,
+                                 TestCredentials.aws_secret_access_key)
+    @key   = 'right_ec2_awesome_test_key'
+    @group = 'right_ec2_awesome_test_security_group'
+  end
+  
+  def test_01_create_describe_key_pairs
+    new_key = @ec2.create_key_pair(@key)
+    assert new_key[:aws_material][/BEGIN RSA PRIVATE KEY/], "New key material is absent"
+    keys = @ec2.describe_key_pairs
+    assert keys.map{|key| key[:aws_key_name] }.include?(@key), "#{@key} must exist"
+  end
+  
+  def test_02_create_security_group
+    assert @ec2.create_security_group(@group,'My awesone test group'), 'Create_security_group fail'
+    group = @ec2.describe_security_groups([@group])[0]
+    assert_equal @group, group[:aws_group_name], 'Group must be created but does not exist'
+  end
+  
+  def test_03_perms_add
+    assert @ec2.authorize_security_group_named_ingress(@group, TestCredentials.account_number, 'default')
+    assert @ec2.authorize_security_group_IP_ingress(@group, 80,80,'udp','192.168.1.0/8')
+  end
+  
+  def test_04_check_new_perms_exist
+    assert_equal 2, @ec2.describe_security_groups([@group])[0][:aws_perms].size
+  end
+
+  def test_05_perms_remove
+    assert @ec2.revoke_security_group_IP_ingress(@group, 80,80,'udp','192.168.1.0/8')
+    assert @ec2.revoke_security_group_named_ingress(@group,
+                                                    TestCredentials.account_number, 'default')
+  end
+
+  def test_06_describe_images
+    images = @ec2.describe_images
+    assert images.size>0, 'Amazon must have at least some public images'
+      # unknown image
+    assert_raise(Rightscale::AwsError){ @ec2.describe_images(['ami-ABCDEFGH'])}
+  end
+
+  def test_07_describe_instanses
+    assert @ec2.describe_instances
+      # unknown image
+    assert_raise(Rightscale::AwsError){ @ec2.describe_instances(['i-ABCDEFGH'])}
+  end
+
+  def test_08_delete_security_group
+    assert @ec2.delete_security_group(@group), 'Delete_security_group fail'
+  end
+  
+  def test_09_delete_key_pair
+    assert @ec2.delete_key_pair(@key), 'Delete_key_pair fail'
+##  Hmmm... Amazon does not through the exception any more. It now just returns a 'true' if the key does not exist any more...
+##      # key must be deleted already
+##    assert_raise(Rightscale::AwsError) { @ec2.delete_key_pair(@key) }
+  end
+
+  def test_10_signature_version_0
+    ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :signature_version => '0')
+    images = ec2.describe_images
+    assert images.size>0, 'Amazon must have at least some public images'
+    # check that the request has correct signature version
+    assert ec2.last_request.path.include?('SignatureVersion=0')
+  end
+
+  def test_11_regions
+    regions = nil
+    assert_nothing_raised do
+      regions = @ec2.describe_regions
+    end
+    # check we got more that 0 regions
+    assert regions.size > 0
+    # check an access to regions
+    regions.each do |region|
+      regional_ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :region => region)
+      # do we have a correct endpoint server?
+      assert_equal "#{region}.ec2.amazonaws.com", regional_ec2.params[:server]
+      # get a list of images from every region
+      images = nil
+      assert_nothing_raised do
+        images = regional_ec2.describe_regions
+      end
+      # every region must have images
+      assert images.size > 0
+    end
+  end
+  
+  def test_12_endpoint_url
+    ec2 = Rightscale::Ec2.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :endpoint_url => 'a://b.c:0/d/', :region => 'z')
+    # :endpoint_url has a priority hence :region should be ommitted
+    assert_equal 'a',   ec2.params[:protocol]
+    assert_equal 'b.c', ec2.params[:server]
+    assert_equal '/d/', ec2.params[:service]
+    assert_equal 0,     ec2.params[:port]
+    assert_nil          ec2.params[:region]
+  end
+
+end

data/test/elb/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../../lib/right_aws'

data/test/elb/test_right_elb.rb

@@ -0,0 +1,43 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestElb < Test::Unit::TestCase
+
+  STDOUT.sync = true
+  BALANCER_NAME = 'right-aws-test-lb'
+
+  def setup
+    @elb = Rightscale::ElbInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, :logger => Logger.new('/dev/null'))
+
+    unless @elb.describe_load_balancers.detect { |lb| lb[:load_balancer_name] == BALANCER_NAME }
+      @elb.create_load_balancer(BALANCER_NAME, %w[us-east-1b], [])
+    end
+    @lb = @elb.describe_load_balancers.detect { |lb| lb[:load_balancer_name] == BALANCER_NAME }
+  end
+
+  # At the end of the day when you want to shut down the test balancer:
+  # * Uncomment this method.
+  # * Comment out all test except one.
+  # * Run this test file.
+  #
+  # def teardown
+  #   @elb.delete_load_balancer BALANCER_NAME
+  # end
+
+  def test_00_describe_load_balancers
+    items = @elb.describe_load_balancers
+    assert items.is_a?(Array)
+  end
+
+  def test_description
+    assert_match /^#{BALANCER_NAME}-\d+\.us-east-1\.elb\.amazonaws\.com$/, @lb[:dns_name]
+  end
+
+  def test_description_has_canonical_hosted_zone_name
+    assert_match /^#{BALANCER_NAME}-\d+\.us-east-1\.elb\.amazonaws\.com$/, @lb[:canonical_hosted_zone_name]
+  end
+
+  def test_description_has_canonical_hosted_zone_name_id
+    assert_match /^[A-Z0-9]+$/, @lb[:canonical_hosted_zone_name_id]
+  end
+
+end