dew 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. data/LICENSE +22 -0
  2. data/README.md +38 -0
  3. data/Rakefile +26 -0
  4. data/bin/dew +87 -0
  5. data/config/cucumber.yaml +4 -0
  6. data/features/create-ami.feature +16 -0
  7. data/features/create-environments.feature +46 -0
  8. data/features/deploy-puge.feature +16 -0
  9. data/features/step_definitions/aws-steps.rb +101 -0
  10. data/features/step_definitions/deploy-puge-steps.rb +27 -0
  11. data/features/support/env.rb +38 -0
  12. data/features/support/hooks.rb +10 -0
  13. data/lib/dew.rb +7 -0
  14. data/lib/dew/aws_resources.yaml +122 -0
  15. data/lib/dew/base_command.rb +24 -0
  16. data/lib/dew/cloud.rb +79 -0
  17. data/lib/dew/commands.rb +6 -0
  18. data/lib/dew/commands/ami.rb +67 -0
  19. data/lib/dew/commands/console.rb +17 -0
  20. data/lib/dew/commands/console/irb_override.rb +24 -0
  21. data/lib/dew/commands/deploy.rb +114 -0
  22. data/lib/dew/commands/deploy/templates/apache.conf.erb +28 -0
  23. data/lib/dew/commands/deploy/templates/known_hosts +2 -0
  24. data/lib/dew/commands/deploy/templates/rvmrc +2 -0
  25. data/lib/dew/commands/environments.rb +110 -0
  26. data/lib/dew/commands/tidy.rb +35 -0
  27. data/lib/dew/controllers.rb +3 -0
  28. data/lib/dew/controllers/amis_controller.rb +82 -0
  29. data/lib/dew/controllers/deploy_controller.rb +10 -0
  30. data/lib/dew/controllers/environments_controller.rb +48 -0
  31. data/lib/dew/models.rb +7 -0
  32. data/lib/dew/models/account.rb +30 -0
  33. data/lib/dew/models/database.rb +32 -0
  34. data/lib/dew/models/deploy.rb +2 -0
  35. data/lib/dew/models/deploy/puge.rb +61 -0
  36. data/lib/dew/models/deploy/run.rb +19 -0
  37. data/lib/dew/models/environment.rb +199 -0
  38. data/lib/dew/models/fog_model.rb +23 -0
  39. data/lib/dew/models/profile.rb +60 -0
  40. data/lib/dew/models/server.rb +134 -0
  41. data/lib/dew/password.rb +7 -0
  42. data/lib/dew/tasks/spec.rake +14 -0
  43. data/lib/dew/validations.rb +8 -0
  44. data/lib/dew/version.rb +3 -0
  45. data/lib/dew/view.rb +39 -0
  46. data/lib/tasks/spec.rake +14 -0
  47. data/spec/dew/cloud_spec.rb +90 -0
  48. data/spec/dew/controllers/amis_controller_spec.rb +137 -0
  49. data/spec/dew/controllers/deploy_controller_spec.rb +38 -0
  50. data/spec/dew/controllers/environments_controller_spec.rb +133 -0
  51. data/spec/dew/models/account_spec.rb +47 -0
  52. data/spec/dew/models/database_spec.rb +58 -0
  53. data/spec/dew/models/deploy/puge_spec.rb +72 -0
  54. data/spec/dew/models/deploy/run_spec.rb +38 -0
  55. data/spec/dew/models/environment_spec.rb +374 -0
  56. data/spec/dew/models/fog_model_spec.rb +24 -0
  57. data/spec/dew/models/profile_spec.rb +85 -0
  58. data/spec/dew/models/server_spec.rb +190 -0
  59. data/spec/dew/password_spec.rb +11 -0
  60. data/spec/dew/spec_helper.rb +22 -0
  61. data/spec/dew/view_spec.rb +38 -0
  62. metadata +284 -0
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ (The MIT License)
2
+
3
+ Copyright (c) 2011 PlayUp
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ 'Software'), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,38 @@
1
+ Dew is a layer between the ground and fog, which is used to access the cloud.
2
+
3
+ It includes:
4
+ * one command `dew`, with subcommands eg. `environments`, `amis`
5
+
6
+ ## Installation
7
+
8
+ First install the dew gem
9
+
10
+ $ [sudo] gem install dew
11
+
12
+ Then create all the config files it needs.
13
+
14
+ $ git clone git@github.com:playup/dew-config.git ~/.dew
15
+
16
+ or
17
+
18
+ $ mkdir -p ~/.dew/accounts
19
+ $ cat > ~/.dew/accounts/development.yaml
20
+ aws:
21
+ user_id: xxxx-xxxx-xxxx
22
+ access_key_id: YYYYYYYYYYYYYYYYYYYY
23
+ secret_access_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ
24
+
25
+
26
+ Then run the dew command
27
+
28
+ $ dew --help
29
+
30
+ ## Creating a Simple Environment
31
+
32
+ ## Creating an AMI for a new Environment
33
+
34
+ ## Deploying to an Environment
35
+
36
+ ## Developing with Dew
37
+
38
+ Read `HACKING.md`
@@ -0,0 +1,26 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ ROOT_DIR = File.expand_path(File.dirname(__FILE__))
5
+
6
+ require 'fileutils'
7
+
8
+ Dir[File.join(ROOT_DIR, 'lib', 'tasks', '*.rake')].each { |task| import task }
9
+
10
+ namespace :metrics do
11
+ desc "Run the flay code duplication measurement over the app, lib and specs"
12
+ task :flay do
13
+ flay_out = File.join(ROOT_DIR, 'reports', 'flay.txt')
14
+ FileUtils.mkdir_p(File.dirname(flay_out)) unless File.directory? File.dirname(flay_out)
15
+ system "flay scripts lib spec > #{flay_out}"
16
+ print "Flay: "
17
+ system "head -n1 #{flay_out}"
18
+ puts "Full flay output in #{flay_out}"
19
+ end
20
+ end
21
+
22
+ task :default => "spec:covered"
23
+
24
+ task :flay do
25
+ sh "flay lib spec script"
26
+ end
data/bin/dew ADDED
@@ -0,0 +1,87 @@
1
+ #!/usr/bin/env ruby
2
+ require 'dew'
3
+
4
+ class DewCommand < DewBaseCommand
5
+ option ['-r', '--region'], "REGION", "AWS region", :default => 'ap-southeast-1'
6
+ option ['-a', '--account'], "ACCOUNT", "AWS account", :default => 'development'
7
+ option ['-q', '--quiet'], :flag, "Quiet mode, disables logging", :default => false
8
+ option ['-v', '--verbose'], :flag, "Verbose mode, print debug output", :default => false
9
+ option '--debug', :flag, "Show stacktraces on error", :default => false
10
+
11
+ subcommand "environments", "perform subcommands on the environments", EnvironmentsCommand
12
+ subcommand "amis", "perform subcommands on the AMIs", AMIsCommand
13
+ subcommand "deploy", "deploy to an environment", DeployCommand
14
+ subcommand "console", "open an IRB session with Cloud loaded", ConsoleCommand
15
+ subcommand "tidy", "tidy up resources left behind by automated tests", TidyCommand
16
+
17
+ def configure
18
+ $debug = debug?
19
+ Inform.level = quiet? ? :warning : (verbose? ? :debug : :info)
20
+ Cloud.connect(region, account)
21
+ end
22
+
23
+ def run(args)
24
+ (puts help ; exit 1) if args.empty?
25
+ super
26
+ end
27
+
28
+
29
+ def execute
30
+ begin
31
+ super
32
+ rescue Clamp::HelpWanted => e
33
+ raise
34
+ rescue Clamp::UsageError => e
35
+ Inform.error(e.message)
36
+ puts e.command.help
37
+ exit(1)
38
+ rescue Interrupt => e
39
+ # If receive ^C Just exit the script...
40
+ Inform.error("Interrupt Received.")
41
+ rescue Exception => e
42
+ error_message = ''
43
+ error_args = {:trace_filename => write_backtrace(e)}
44
+
45
+ if e.is_a? Excon::Errors::HTTPStatusError
46
+ error_args[:excon] = parse_excon_error_response(e)
47
+ error_message = "AWS Error: %{excon}"
48
+ else
49
+ error_message += e.message
50
+ end
51
+ error_message += "\n(see %{trace_filename} for more details)\n"
52
+
53
+ if $debug or ENV['AWS_DEBUG']
54
+ Inform.error(error_message, error_args)
55
+ raise
56
+ else
57
+ error_message += "(and/or run again with %{flag} flag, or %{env} in your environment)\n"
58
+ error_args.merge!(:flag => "--debug", :env => "AWS_DEBUG=1")
59
+ Inform.error(error_message, error_args)
60
+ Kernel.exit 1
61
+ end
62
+ end
63
+ end
64
+
65
+ private
66
+
67
+ def parse_excon_error_response e
68
+ begin
69
+ require 'nokogiri'
70
+ Nokogiri::XML.parse(e.response.body).css('Message').first.content
71
+ rescue
72
+ e.message
73
+ end
74
+ end
75
+
76
+ def write_backtrace e
77
+ trace_filename = "/tmp/#{invocation_path}-#{Time.now.strftime("%Y%m%d%H%M%S")}.txt"
78
+ File.open(trace_filename, 'w') do |f|
79
+ backtrace = (["#{e.backtrace.shift}: #{e.message} (#{e.class})"] + e.backtrace).join("\n\t")
80
+ f.write("#{self.inspect}\n\n#{backtrace}\n")
81
+ f.close
82
+ end
83
+ trace_filename
84
+ end
85
+ end
86
+
87
+ DewCommand.run
@@ -0,0 +1,4 @@
1
+ default: features -t~@wip -t~@slow
2
+ slow: features -t~@wip -t@slow
3
+ wip: features -t@wip
4
+ all: features -t~@wip
@@ -0,0 +1,16 @@
1
+ Feature: Creating a new AMI
2
+
3
+ As PlayUp
4
+ I want new AMIs to be created from Puppet configurations
5
+ So that I can use those AMIs to create new environments
6
+
7
+ @aws
8
+ @slow
9
+ Scenario: Creating a new AMI from a puppet configuration
10
+ Given I specify the puppet configuration "blank"
11
+ And I uniquely name my AMI
12
+ And I specify the region "ap-southeast-1" and account "development"
13
+ When I run the create-ami script
14
+ Then the script should report success
15
+ And the script should return an AMI id
16
+ And that AMI should exist as a private AMI in our AWS account
@@ -0,0 +1,46 @@
1
+ Feature: Creating a new environment
2
+
3
+ As PlayUp
4
+ I want new environments to be provisioned automatically
5
+ So that I can deploy applications to those environments
6
+
7
+ @aws
8
+ Scenario: Creating a new light environment using the development account
9
+ Given I specify the environment profile "test-light"
10
+ And I uniquely name my environment
11
+ And I specify the region "ap-southeast-1" and account "development"
12
+ When I run the create-environment script
13
+ Then the script should report success
14
+ And I should have "2" running EC2 servers
15
+ And the server names should be prefixed with the environment name
16
+ And the servers should be in the "ap-southeast" availability zone
17
+ And the servers should be tagged with "Name"
18
+ And the servers should be tagged with "Environment"
19
+ And the servers should be tagged with "Creator"
20
+ And I should be able to SSH in to each server using the "devops" keypair
21
+ And there should be a load balancer in front of the servers
22
+
23
+ @slow
24
+ @aws
25
+ Scenario Outline: Creating a new full environment
26
+ Given I specify the environment profile "test-full"
27
+ And I uniquely name my environment
28
+ And I specify the region "<region>" and account "development"
29
+ When I run the create-environment script
30
+ Then the script should report success
31
+ And I should have "2" running EC2 servers
32
+ And the server names should be prefixed with the environment name
33
+ And the servers should be in the "<region>" availability zone
34
+ And the servers should be tagged with "Name"
35
+ And the servers should be tagged with "Environment"
36
+ And the servers should be tagged with "Creator"
37
+ And I should be able to SSH in to each server using the "devops" keypair
38
+ And there should be a load balancer in front of the servers
39
+ And I should have an RDS for my environment
40
+ And PUGE database environment variables should be set on each server to match the created RDS
41
+ And I should be able to connect to the RDS
42
+
43
+ Scenarios: regions
44
+ | region |
45
+ | ap-southeast-1 |
46
+ | eu-west-1 |
@@ -0,0 +1,16 @@
1
+ Feature: Deploying Puge to an environment
2
+
3
+ As PlayUp
4
+ I want to deploy Puge to a new environment
5
+ So that I can test it
6
+
7
+ @wip
8
+ @aws
9
+ @slow
10
+ Scenario: Deploying Puge from Gitub to a new environment
11
+ Given an environment that PUGE can be deployed to
12
+ And I run the create-environment script
13
+ And I specify the Git PUGE tag "master"
14
+ When I run the deployment script
15
+ Then the script should report success
16
+ And I should see the correct PUGE tag has been deployed when I hit the load balancer
@@ -0,0 +1,101 @@
1
+ Given /^I specify the environment profile "([^"]*)"$/ do |profile|
2
+ @profile = profile
3
+ end
4
+
5
+ Given /^I uniquely name my environment$/ do
6
+ @environment_name = unique_test_name
7
+ end
8
+
9
+ Given /^I uniquely name my AMI$/ do
10
+ @ami_name = unique_test_name
11
+ end
12
+
13
+ Given /^I specify the region "([^"]*)" and account "([^"]*)"$/ do |region, account_name|
14
+ @region = region
15
+ @account_name = account_name
16
+ Cloud.connect(@region, @account_name)
17
+ end
18
+
19
+ Given /^I specify the puppet configuration "([^"]*)"$/ do |puppet_config|
20
+ @puppet_config = puppet_config
21
+ end
22
+
23
+ When /^I run the create\-environment script$/ do
24
+ @log = run_and_capture("./bin/dew --region #{@region} --account #{@account_name} --debug --verbose environments create -f #{@profile} #{@environment_name}", "create-environment.#{@profile}")
25
+ end
26
+
27
+ When /^I run the create\-ami script$/ do
28
+ @log = run_and_capture("./bin/dew --region #{@region} --account #{@account_name} --debug --verbose amis create #{@puppet_config} #{@ami_name}", "create-ami")
29
+ end
30
+
31
+ Then /^the script should report success$/ do
32
+ if $?.exitstatus != 0 && File.exist?(@log)
33
+ $stderr.puts File.read(@log)
34
+ end
35
+ $?.exitstatus.should == 0
36
+ end
37
+
38
+ Then /^the script should return an AMI id$/ do
39
+ script_output = File.read(@log)
40
+ script_output.should match(/AMI ID is ([^ ]+)/i)
41
+ script_output =~ /AMI ID is.+(ami-[\w]+)/i # damn matcher won't capture :(
42
+ @ami_id = $1
43
+ end
44
+
45
+ Then /^that AMI should exist as a private AMI in our AWS account$/ do
46
+ ami = Cloud.compute.images.all('name' => @ami_name).first
47
+ @ami_id.should == ami.id
48
+ ami.is_public.should be_false
49
+ end
50
+
51
+ Then /^I should have "([^"]*)" running EC2 servers$/ do |number_of_servers|
52
+ environment.servers.length.should == number_of_servers.to_i
53
+ end
54
+
55
+ Then /^the server names should be prefixed with the environment name$/ do
56
+ server_names = environment.servers.collect { |server| server.tags["Name"] }
57
+ server_names.sort.should == ["#{@environment_name} 1", "#{@environment_name} 2"]
58
+ end
59
+
60
+ Then /^the servers should be in the "([^"]*)" availability zone$/ do |availability_zone|
61
+ environment.servers.each { |server| server.availability_zone.should match /^#{availability_zone}(.*)$/ }
62
+ end
63
+
64
+ Then /^the servers should be tagged with "([^"]*)"$/ do |tag|
65
+ environment.servers.each { |server| server.tags.should have_key tag }
66
+ end
67
+
68
+ Then /^there should be a load balancer in front of the servers$/ do
69
+ servers = elb['Instances']
70
+
71
+ servers.sort.should == environment.servers.collect { |server| server.id }.sort
72
+ end
73
+
74
+ Then /^I should have an RDS for my environment$/ do
75
+ environment.database.should_not be_nil
76
+ end
77
+
78
+ Then /^I should be able to SSH in to each server using the "([^"]*)" keypair$/ do |keypair|
79
+ environment.servers.each do |server|
80
+ server.ssh.run("echo hello").should == "hello\n"
81
+ end
82
+ end
83
+
84
+ Then /^PUGE database environment variables should be set on each server to match the created RDS$/ do
85
+ environment.servers.each do |server|
86
+ ssh = server.ssh
87
+ %w{PUGE_DB_NAME PUGE_DB_USERNAME PUGE_DB_PASSWORD}.each do |var|
88
+ ssh.run("echo $#{var}").chomp.length.should_not == 0
89
+ end
90
+ ssh.run("echo $PUGE_DB_HOST").chomp.should == environment.database.public_address
91
+ end
92
+ end
93
+
94
+ Then /^I should be able to connect to the RDS$/ do
95
+ environment.servers.each do |server|
96
+ ssh = server.ssh
97
+ #host.run("nc -w 1 $PUGE_DB_HOST 3306 > /dev/null 2>&1 ; echo $?").chomp.should == "0"
98
+ ssh.run("sudo apt-get install -y mysql-client", :quiet_stderr => true)
99
+ ssh.run("echo show databases | mysql -u$PUGE_DB_USERNAME -p$PUGE_DB_PASSWORD -h$PUGE_DB_HOST").chomp.length.should_not == 0
100
+ end
101
+ end
@@ -0,0 +1,27 @@
1
+ Given /^an environment that PUGE can be deployed to$/ do
2
+ Given 'I specify the environment profile "puge-deploy"'
3
+ And "I uniquely name my environment"
4
+ And 'I specify the region "ap-southeast-1" and account "development"'
5
+ end
6
+
7
+ Given /^I specify the Git PUGE tag "([^"]*)"$/ do |tag|
8
+ @tag = tag
9
+ end
10
+
11
+ When /^I run the deployment script$/ do
12
+ @log = run_and_capture("script/deploy.rb puge --region #{Cloud.region} --account #{Cloud.account_name} #{@tag} #{@environment_name} development", "deploy")
13
+ sleep 60 # wait for tomcat to restart
14
+ end
15
+
16
+ Then /^I should see the correct PUGE tag has been deployed when I hit the load balancer$/ do
17
+ require 'net/http'
18
+
19
+ http = Net::HTTP.new(elb['DNSName'], 80)
20
+ http.read_timeout = 60
21
+
22
+ response = http.start do |web|
23
+ web.request(Net::HTTP::Get.new('/admin'))
24
+ end
25
+
26
+ response.body.should =~ /Revision: \w{7} /
27
+ end
@@ -0,0 +1,38 @@
1
+ require 'timeout'
2
+ require 'pathname'
3
+
4
+ ROOT_DIR = Pathname(__FILE__).expand_path.parent.parent.parent
5
+ $: << (ROOT_DIR+'lib').to_s
6
+ require 'dew'
7
+
8
+ def tmp_dir
9
+ tmp_dir = ROOT_DIR.join('tmp')
10
+ tmp_dir.mkdir unless tmp_dir.exist?
11
+ tmp_dir
12
+ end
13
+
14
+ def run_and_capture command, logname, timeout = 1200
15
+ logfile = tmp_dir.join(logname + ".log")
16
+ begin
17
+ Timeout::timeout(timeout) do
18
+ system("RUBYLIB='lib' #{command} > '#{logfile}' 2>&1")
19
+ end
20
+ rescue Timeout::Error => e
21
+ $stderr.puts "*** Command #{command} took longer than #{timeout} seconds to run, log follows:"
22
+ $stderr.puts logfile.read
23
+ raise e
24
+ end
25
+ logfile
26
+ end
27
+
28
+ def unique_test_name
29
+ "cuke-#{ENV['USER']}-#{Time.now.to_i}"
30
+ end
31
+
32
+ def environment
33
+ @environment ||= Environment.get(@environment_name) if @environment_name
34
+ end
35
+
36
+ def elb
37
+ @elb ||= Cloud.elb.describe_load_balancers(@environment_name).body['DescribeLoadBalancersResult']['LoadBalancerDescriptions'].first
38
+ end
@@ -0,0 +1,10 @@
1
+ After("@aws") do
2
+ if !ENV['KEEP_TEST_ARTIFACTS']
3
+ if @environment_name
4
+ run_and_capture("./bin/dew --region #{@region} --account #{@account_name} environments destroy -f #{@environment_name}", 'destroy-environment')
5
+ end
6
+ if @ami_name
7
+ run_and_capture("./bin/dew --region #{@region} --account #{@account_name} amis destroy -f #{@ami_name}", 'destroy-ami')
8
+ end
9
+ end
10
+ end