dew 0.1.0

data/LICENSE

@@ -0,0 +1,22 @@
+(The MIT License)
+
+Copyright (c) 2011 PlayUp
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+Dew is a layer between the ground and fog, which is used to access the cloud.
+
+It includes:
+  * one command `dew`, with subcommands eg. `environments`, `amis`
+
+## Installation
+
+First install the dew gem
+
+    $ [sudo] gem install dew
+
+Then create all the config files it needs.
+
+    $ git clone git@github.com:playup/dew-config.git ~/.dew
+
+or
+
+    $ mkdir -p ~/.dew/accounts
+    $ cat > ~/.dew/accounts/development.yaml
+    aws:
+      user_id: xxxx-xxxx-xxxx
+      access_key_id: YYYYYYYYYYYYYYYYYYYY
+      secret_access_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ
+
+
+Then run the dew command
+
+    $ dew --help
+
+## Creating a Simple Environment
+
+## Creating an AMI for a new Environment
+
+## Deploying to an Environment
+
+## Developing with Dew
+
+Read `HACKING.md`

data/Rakefile

@@ -0,0 +1,26 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+ROOT_DIR = File.expand_path(File.dirname(__FILE__))
+
+require 'fileutils'
+
+Dir[File.join(ROOT_DIR, 'lib', 'tasks', '*.rake')].each { |task| import task }
+
+namespace :metrics do
+  desc "Run the flay code duplication measurement over the app, lib and specs"
+  task :flay do
+    flay_out = File.join(ROOT_DIR, 'reports', 'flay.txt')
+    FileUtils.mkdir_p(File.dirname(flay_out)) unless File.directory? File.dirname(flay_out)
+    system "flay scripts lib spec > #{flay_out}"
+    print "Flay: "
+    system "head -n1 #{flay_out}"
+    puts "Full flay output in #{flay_out}"
+  end
+end
+
+task :default => "spec:covered"
+
+task :flay do
+  sh "flay lib spec script"
+end

data/bin/dew

@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby
+require 'dew'
+
+class DewCommand < DewBaseCommand
+  option ['-r', '--region'], "REGION", "AWS region", :default => 'ap-southeast-1'
+  option ['-a', '--account'], "ACCOUNT", "AWS account", :default => 'development'
+  option ['-q', '--quiet'], :flag, "Quiet mode, disables logging", :default => false
+  option ['-v', '--verbose'], :flag, "Verbose mode, print debug output", :default => false
+  option '--debug', :flag, "Show stacktraces on error", :default => false
+
+  subcommand "environments", "perform subcommands on the environments", EnvironmentsCommand
+  subcommand "amis", "perform subcommands on the AMIs", AMIsCommand
+  subcommand "deploy", "deploy to an environment", DeployCommand
+  subcommand "console", "open an IRB session with Cloud loaded", ConsoleCommand
+  subcommand "tidy", "tidy up resources left behind by automated tests", TidyCommand
+
+  def configure
+    $debug = debug?
+    Inform.level = quiet? ? :warning : (verbose? ? :debug : :info)
+    Cloud.connect(region, account)
+  end
+
+  def run(args)
+    (puts help ; exit 1) if args.empty?
+    super
+  end
+
+
+  def execute
+    begin
+      super
+    rescue Clamp::HelpWanted => e
+      raise
+    rescue Clamp::UsageError => e
+      Inform.error(e.message)
+      puts e.command.help
+      exit(1)
+    rescue Interrupt => e
+      # If receive ^C Just exit the script...
+      Inform.error("Interrupt Received.")
+    rescue Exception => e
+      error_message = ''
+      error_args = {:trace_filename => write_backtrace(e)}
+      
+      if e.is_a? Excon::Errors::HTTPStatusError
+        error_args[:excon] = parse_excon_error_response(e)
+        error_message = "AWS Error: %{excon}"        
+      else
+        error_message += e.message
+      end
+      error_message += "\n(see %{trace_filename} for more details)\n"
+
+      if $debug or ENV['AWS_DEBUG']
+        Inform.error(error_message, error_args)
+        raise
+      else
+        error_message += "(and/or run again with %{flag} flag, or %{env} in your environment)\n"
+        error_args.merge!(:flag => "--debug", :env => "AWS_DEBUG=1")
+        Inform.error(error_message, error_args)
+        Kernel.exit 1
+      end
+    end
+  end
+  
+  private
+  
+  def parse_excon_error_response e
+    begin
+      require 'nokogiri'
+      Nokogiri::XML.parse(e.response.body).css('Message').first.content
+    rescue
+      e.message
+    end
+  end
+
+  def write_backtrace e
+    trace_filename = "/tmp/#{invocation_path}-#{Time.now.strftime("%Y%m%d%H%M%S")}.txt"
+    File.open(trace_filename, 'w') do |f|
+      backtrace = (["#{e.backtrace.shift}: #{e.message} (#{e.class})"] + e.backtrace).join("\n\t")
+      f.write("#{self.inspect}\n\n#{backtrace}\n")
+      f.close
+    end
+    trace_filename
+  end
+end
+
+DewCommand.run

data/config/cucumber.yaml

@@ -0,0 +1,4 @@
+default: features -t~@wip -t~@slow
+slow: features -t~@wip -t@slow
+wip: features -t@wip
+all: features -t~@wip

data/features/create-ami.feature

@@ -0,0 +1,16 @@
+Feature: Creating a new AMI
+
+  As PlayUp
+  I want new AMIs to be created from Puppet configurations
+  So that I can use those AMIs to create new environments
+
+  @aws
+  @slow
+  Scenario: Creating a new AMI from a puppet configuration
+    Given I specify the puppet configuration "blank"
+    And I uniquely name my AMI
+    And I specify the region "ap-southeast-1" and account "development"
+    When I run the create-ami script
+    Then the script should report success
+    And the script should return an AMI id
+    And that AMI should exist as a private AMI in our AWS account

data/features/create-environments.feature

@@ -0,0 +1,46 @@
+Feature: Creating a new environment
+
+  As PlayUp
+  I want new environments to be provisioned automatically
+  So that I can deploy applications to those environments
+
+  @aws
+  Scenario: Creating a new light environment using the development account
+    Given I specify the environment profile "test-light"
+    And I uniquely name my environment
+    And I specify the region "ap-southeast-1" and account "development"
+    When I run the create-environment script
+    Then the script should report success
+    And I should have "2" running EC2 servers
+    And the server names should be prefixed with the environment name
+    And the servers should be in the "ap-southeast" availability zone
+    And the servers should be tagged with "Name"
+    And the servers should be tagged with "Environment"
+    And the servers should be tagged with "Creator"
+    And I should be able to SSH in to each server using the "devops" keypair
+    And there should be a load balancer in front of the servers
+
+  @slow
+  @aws
+  Scenario Outline: Creating a new full environment
+    Given I specify the environment profile "test-full"
+    And I uniquely name my environment
+    And I specify the region "<region>" and account "development"
+    When I run the create-environment script
+    Then the script should report success
+    And I should have "2" running EC2 servers
+    And the server names should be prefixed with the environment name
+    And the servers should be in the "<region>" availability zone
+    And the servers should be tagged with "Name"
+    And the servers should be tagged with "Environment"
+    And the servers should be tagged with "Creator"
+    And I should be able to SSH in to each server using the "devops" keypair
+    And there should be a load balancer in front of the servers
+    And I should have an RDS for my environment
+    And PUGE database environment variables should be set on each server to match the created RDS
+    And I should be able to connect to the RDS
+
+    Scenarios: regions
+      | region         |
+      | ap-southeast-1 |
+      | eu-west-1      |

data/features/deploy-puge.feature

@@ -0,0 +1,16 @@
+Feature: Deploying Puge to an environment
+
+ As PlayUp
+ I want to deploy Puge to a new environment
+ So that I can test it
+
+ @wip
+ @aws
+ @slow
+ Scenario: Deploying Puge from Gitub to a new environment
+   Given an environment that PUGE can be deployed to
+   And I run the create-environment script
+   And I specify the Git PUGE tag "master"
+   When I run the deployment script
+   Then the script should report success
+   And I should see the correct PUGE tag has been deployed when I hit the load balancer

data/features/step_definitions/aws-steps.rb

@@ -0,0 +1,101 @@
+Given /^I specify the environment profile "([^"]*)"$/ do |profile|
+  @profile = profile
+end
+
+Given /^I uniquely name my environment$/ do
+  @environment_name = unique_test_name 
+end
+
+Given /^I uniquely name my AMI$/ do
+  @ami_name = unique_test_name
+end
+
+Given /^I specify the region "([^"]*)" and account "([^"]*)"$/ do |region, account_name|
+  @region = region
+  @account_name = account_name
+  Cloud.connect(@region, @account_name)
+end
+
+Given /^I specify the puppet configuration "([^"]*)"$/ do |puppet_config|
+  @puppet_config = puppet_config
+end
+
+When /^I run the create\-environment script$/ do
+  @log = run_and_capture("./bin/dew --region #{@region} --account #{@account_name} --debug --verbose environments create -f #{@profile} #{@environment_name}", "create-environment.#{@profile}")
+end
+
+When /^I run the create\-ami script$/ do
+  @log = run_and_capture("./bin/dew --region #{@region} --account #{@account_name} --debug --verbose amis create #{@puppet_config} #{@ami_name}", "create-ami")
+end
+
+Then /^the script should report success$/ do
+  if $?.exitstatus != 0 && File.exist?(@log)
+    $stderr.puts File.read(@log)
+  end
+  $?.exitstatus.should == 0
+end
+
+Then /^the script should return an AMI id$/ do
+  script_output = File.read(@log)
+  script_output.should match(/AMI ID is ([^ ]+)/i)
+  script_output =~ /AMI ID is.+(ami-[\w]+)/i # damn matcher won't capture :(
+  @ami_id = $1
+end
+
+Then /^that AMI should exist as a private AMI in our AWS account$/ do
+  ami = Cloud.compute.images.all('name' => @ami_name).first
+  @ami_id.should == ami.id
+  ami.is_public.should be_false
+end
+
+Then /^I should have "([^"]*)" running EC2 servers$/ do |number_of_servers|
+  environment.servers.length.should == number_of_servers.to_i
+end
+
+Then /^the server names should be prefixed with the environment name$/ do
+  server_names = environment.servers.collect { |server| server.tags["Name"] }
+  server_names.sort.should == ["#{@environment_name} 1", "#{@environment_name} 2"]
+end
+
+Then /^the servers should be in the "([^"]*)" availability zone$/ do |availability_zone|
+  environment.servers.each { |server| server.availability_zone.should match /^#{availability_zone}(.*)$/ }
+end
+
+Then /^the servers should be tagged with "([^"]*)"$/ do |tag|
+  environment.servers.each { |server| server.tags.should have_key tag }
+end
+
+Then /^there should be a load balancer in front of the servers$/ do
+  servers = elb['Instances']
+  
+  servers.sort.should == environment.servers.collect { |server| server.id }.sort
+end
+
+Then /^I should have an RDS for my environment$/ do
+  environment.database.should_not be_nil
+end
+
+Then /^I should be able to SSH in to each server using the "([^"]*)" keypair$/ do |keypair|
+  environment.servers.each do |server|
+    server.ssh.run("echo hello").should == "hello\n"
+  end
+end
+
+Then /^PUGE database environment variables should be set on each server to match the created RDS$/ do
+  environment.servers.each do |server|
+    ssh = server.ssh
+    %w{PUGE_DB_NAME PUGE_DB_USERNAME PUGE_DB_PASSWORD}.each do |var|
+      ssh.run("echo $#{var}").chomp.length.should_not == 0
+    end
+    ssh.run("echo $PUGE_DB_HOST").chomp.should == environment.database.public_address
+  end
+end
+
+Then /^I should be able to connect to the RDS$/ do
+  environment.servers.each do |server|
+    ssh = server.ssh
+    #host.run("nc -w 1 $PUGE_DB_HOST 3306 > /dev/null 2>&1 ; echo $?").chomp.should == "0"
+    ssh.run("sudo apt-get install -y mysql-client", :quiet_stderr => true)
+    ssh.run("echo show databases | mysql -u$PUGE_DB_USERNAME -p$PUGE_DB_PASSWORD -h$PUGE_DB_HOST").chomp.length.should_not == 0
+  end
+end

data/features/step_definitions/deploy-puge-steps.rb

@@ -0,0 +1,27 @@
+Given /^an environment that PUGE can be deployed to$/ do
+  Given 'I specify the environment profile "puge-deploy"'
+  And "I uniquely name my environment"
+  And 'I specify the region "ap-southeast-1" and account "development"'
+end
+
+Given /^I specify the Git PUGE tag "([^"]*)"$/ do |tag|
+  @tag = tag
+end
+
+When /^I run the deployment script$/ do
+  @log = run_and_capture("script/deploy.rb puge --region #{Cloud.region} --account #{Cloud.account_name} #{@tag} #{@environment_name} development", "deploy")
+  sleep 60    # wait for tomcat to restart
+end
+
+Then /^I should see the correct PUGE tag has been deployed when I hit the load balancer$/ do
+  require 'net/http'
+
+  http = Net::HTTP.new(elb['DNSName'], 80)
+  http.read_timeout = 60
+
+  response = http.start do |web|
+    web.request(Net::HTTP::Get.new('/admin'))
+  end
+
+  response.body.should =~ /Revision: \w{7} /
+end

data/features/support/env.rb

@@ -0,0 +1,38 @@
+require 'timeout'
+require 'pathname'
+
+ROOT_DIR = Pathname(__FILE__).expand_path.parent.parent.parent
+$: << (ROOT_DIR+'lib').to_s
+require 'dew'
+
+def tmp_dir
+  tmp_dir = ROOT_DIR.join('tmp')
+  tmp_dir.mkdir unless tmp_dir.exist?
+  tmp_dir
+end
+
+def run_and_capture command, logname, timeout = 1200
+  logfile = tmp_dir.join(logname + ".log")
+  begin
+    Timeout::timeout(timeout) do
+      system("RUBYLIB='lib' #{command} > '#{logfile}' 2>&1")
+    end
+  rescue Timeout::Error => e
+    $stderr.puts "*** Command #{command} took longer than #{timeout} seconds to run, log follows:"
+    $stderr.puts logfile.read
+    raise e
+  end
+  logfile
+end
+
+def unique_test_name
+  "cuke-#{ENV['USER']}-#{Time.now.to_i}"
+end
+
+def environment
+  @environment ||= Environment.get(@environment_name) if @environment_name
+end
+
+def elb
+  @elb ||= Cloud.elb.describe_load_balancers(@environment_name).body['DescribeLoadBalancersResult']['LoadBalancerDescriptions'].first
+end

data/features/support/hooks.rb

@@ -0,0 +1,10 @@
+After("@aws") do
+  if !ENV['KEEP_TEST_ARTIFACTS']
+    if @environment_name
+      run_and_capture("./bin/dew --region #{@region} --account #{@account_name} environments destroy -f #{@environment_name}", 'destroy-environment')
+    end
+    if @ami_name
+      run_and_capture("./bin/dew --region #{@region} --account #{@account_name} amis destroy -f #{@ami_name}", 'destroy-ami')
+    end
+  end
+end