ec2launcher 1.3.10 → 1.3.12

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 1.3.12
+
+* Use run_with_backoff when searching for AMI.
+* Fixed bug introduced by BlockDeviceBuilder refactoring.
+
+## 1.3.11
+
+* Fixed logging bug in BlockDeviceBuilder (again).
+
+## 1.3.10
+
+* Made additional methods in BlockDeviceBuilder public for reuse elsewhere.
+* Fixed small bug in BlockDeviceBuilder when unable to retrieve existing Logger.
+
 ## 1.3.9
 
 * More cleanup to terminator code.

data/README.md

@@ -1,4 +1,7 @@
-# ec2launcher
+# EC2Launcher
+[![Build Status](https://secure.travis-ci.org/StudyBlue/ec2launcher.png)](http://travis-ci.org/StudyBlue/ec2launcher)
+[![Dependency Status](https://gemnasium.com/StudyBlue/ec2launcher.png)](https://gemnasium.com/StudyBlue/ec2launcher)
+[![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/StudyBlue/ec2launcher)
 
 A tool to help launch Amazon EC2 instances.
 
@@ -13,4 +16,8 @@ All contributions are welcome: ideas, patches, documentation, bug reports, compl
 
 # Support
 
-[ec2launcher Google Groups](http://groups.google.com/group/ec2launcher-user)
+[ec2launcher Google Groups](http://groups.google.com/group/ec2launcher-user)
+
+# Authors and Contributors
+
+* Sean Laurent

data/Rakefile

@@ -1,2 +1,9 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << 'lib/ec2launcher'
+  t.test_files = FileList['test/ec2launcher/*_test.rb', "test/ec2launcher/dsl/*_test.rb"]
+  t.verbose = true
+end
+task :default => :test

data/ec2launcher.gemspec

@@ -16,4 +16,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "aws-sdk", [">= 1.6.6"]
   gem.add_runtime_dependency "log4r"
+
+  gem.add_development_dependency "minitest"
+  gem.add_development_dependency "rake"
 end

data/lib/ec2launcher.rb

@@ -164,11 +164,8 @@ module EC2Launcher
       sg_map = { }
       AWS.start_memoizing
       @ec2.security_groups.each do |sg|
-        if ec2_subnet.nil?
-          next unless sg.vpc_id.nil?
-        else
-          next unless ec2_subnet.vpc_id == sg.vpc_id
-        end
+        next if ec2_subnet.nil? && sg.vpc_id
+        next if ec2_subnet && ec2_subnet.vpc_id != sg.vpc_id
         sg_map[sg.name] = sg
       end
       AWS.stop_memoizing
@@ -178,6 +175,7 @@ module EC2Launcher
       missing_security_groups = []
       security_groups.each do |sg_name|
         missing_security_groups << sg_name unless sg_map.has_key?(sg_name)
+        puts sg_name
         security_group_ids << sg_map[sg_name].security_group_id
       end
 
@@ -216,7 +214,10 @@ module EC2Launcher
       ##############################
       ami_name_match = @application.ami_name
       ami_name_match ||= @environment.ami_name
-      ami = find_ami(instance_architecture, instance_virtualization, ami_name_match, @options.ami_id)
+      ami = nil
+      run_with_backoff(60, 1, "searching for ami") do
+        ami = find_ami(instance_architecture, instance_virtualization, ami_name_match, @options.ami_id)
+      end
 
       ##############################
       # DOMAIN NAME

data/lib/ec2launcher/block_device_builder.rb

@@ -2,6 +2,7 @@
 # Copyright (c) 2012 Sean Laurent
 #
 require 'ec2launcher/defaults'
+require 'ec2launcher/backoff_runner'
 require 'log4r'
 
 include Log4r
@@ -10,6 +11,8 @@ module EC2Launcher
   # Helper class to build EC2 block device definitions.
   #
   class BlockDeviceBuilder
+    include BackoffRunner
+
     attr_reader :block_device_mappings
     attr_reader :block_device_tags
 
@@ -25,10 +28,11 @@ module EC2Launcher
       @block_device_tags = {}
 
       begin
-        @log = Logger['ec2launcher']
+        @log = Log4r::Logger['ec2launcher']
       rescue
-        @log = Logger.new 'ec2launcher'
       end
+
+      @log ||= Log4r::Logger.new 'ec2launcher'
     end
 
     # Generates the mappings for ephemeral and ebs volumes.
@@ -177,37 +181,38 @@ module EC2Launcher
       AWS.stop_memoizing
     end
 
-    public
-    # Retrieves the latest set of completed snapshots for a RAID array of EBS volumes.
-    #
-    # Volumes must have the following tags:
-    #   * host
-    #   * volumeName
-    #   * time
-    #
-    # @param [String] hostname FQDN for new host
+    private
     # @param [String] purpose purpose of RAID array, which is stored in the `purpose` tag for snapshots/volumes
     #        and is part of the snapshot name.
-    # @param [Integer] count number of EBS snapshots to look for
+    # @param [AWS::EC2::SnapshotCollection] snapshots collection of snapshots to examine
     #
-    # @return [Hash<Integer, AWS::EC2::Snapshot>] mapping of RAID device numbers (zero based) to AWS Snapshots.
-    #
-    def get_latest_raid_snapshot_mapping(hostname, purpose, count)
-      @log.info "Retrieving list of snapshots ..."
-      result = @ec2.snapshots.tagged("host").tagged_values(hostname).tagged("volumeName").tagged_values("*raid*").tagged("time")
-
-      @log.info "Building list of snapshots to clone (#{purpose}) for '#{hostname}'..."
+    # @return [Array<AWS::EC2::Snapshot>] list of matching snapshots
+    def build_snapshot_clone_list(purpose, snapshots)
       snapshot_name_regex = /#{purpose} raid.*/
       host_snapshots = []
-      result.each do |s|
-        next if snapshot_name_regex.match(s.tags["volumeName"]).nil?
-        host_snapshots << s if s.status == :completed
+      snapshots.each do |s|
+        name_no_match = false
+        run_with_backoff(60, 1, "searching for snapshots") do
+          name_no_match = snapshot_name_regex.match(s.tags["volumeName"]).nil?
+        end
+        next if name_no_match
+
+        run_with_backoff(60, 1, "searching for snapshots") do
+          host_snapshots << s if s.status == :completed
+        end
       end
+      host_snapshots
+    end
 
-      # Bucket the snapshots based on volume number e.g. "raid (1)" vs "raid (2)"
+    private
+    # Creates a mapping of volume numbers to snapshots.
+    #
+    # @param [Array<AWS::EC2::Snapshot>] snapshots list of matching snapshots
+    # @result [Hash<String, AWS::EC2::Snapshot>] map of volume numbers as strings  ("1", "2", etc.) to snapshots
+    def bucket_snapshots_by_volume_number(snapshots)
       snapshot_buckets = { }
       volume_number_regex = /raid \((\d)\)$/
-      host_snapshots.each do |snapshot|
+      snapshots.each do |snapshot|
         next if snapshot.tags["time"].nil?
 
         volume_name = snapshot.tags["volumeName"]
@@ -223,14 +228,14 @@ module EC2Launcher
         snapshot_buckets[raid_number] = [] if snapshot_buckets[raid_number].nil?
         snapshot_buckets[raid_number] << snapshot
       end
+      snapshot_buckets
+    end
 
-      # Sort the snapshots in each bucket by time
-      snapshot_buckets.keys.each do |key|
-        snapshot_buckets[key].sort! do |a, b|
-          b.tags["time"] <=> a.tags["time"]
-        end
-      end
-
+    private
+    # Find the most recent backup time that all snapshots have in common.
+    #
+    # @param [Hash<String, AWS::EC2::Snapshot>] snapshot_buckets map of volume numbers as strings  ("1", "2", etc.) to snapshots
+    def get_most_recent_common_snapshot_time(snapshot_buckets)
       # We need to find the most recent backup time that all snapshots have in common.
       #
       # For example, we may have the following snapshots for "raid (1)":
@@ -251,15 +256,54 @@ module EC2Launcher
         most_recent_dates << snapshot.tags["time"].to_s
       end
       most_recent_dates.sort!
+      most_recent_dates[0]
+    end
+
+    public
+    # Retrieves the latest set of completed snapshots for a RAID array of EBS volumes.
+    #
+    # Volumes must have the following tags:
+    #   * host
+    #   * volumeName
+    #   * time
+    #
+    # @param [String] hostname FQDN for new host
+    # @param [String] purpose purpose of RAID array, which is stored in the `purpose` tag for snapshots/volumes
+    #        and is part of the snapshot name.
+    # @param [Integer] count number of EBS snapshots to look for
+    #
+    # @return [Hash<Integer, AWS::EC2::Snapshot>] mapping of RAID device numbers (zero based) to AWS Snapshots.
+    #
+    def get_latest_raid_snapshot_mapping(hostname, purpose, count)
+      @log.info "Retrieving list of snapshots ..."
+      result = @ec2.snapshots.tagged("host").tagged_values(hostname).tagged("volumeName").tagged_values("*raid*").tagged("time")
+
+      @log.info "Building list of snapshots to clone (#{purpose}) for '#{hostname}'..."
+      snapshot_name_regex = /#{purpose} raid.*/
+      host_snapshots = build_snapshot_clone_list(purpose, result)
+
+      # Bucket the snapshots based on volume number e.g. "raid (1)" vs "raid (2)"
+      snapshot_buckets = bucket_snapshots_by_volume_number(host_snapshots)
+
+      # Sort the snapshots in each bucket by time
+      snapshot_buckets.keys.each do |key|
+        snapshot_buckets[key].sort! do |a, b|
+          b.tags["time"] <=> a.tags["time"]
+        end
+      end
+
+      # We need to find the most recent backup time that all snapshots have in common.
+      most_recent_date = get_most_recent_common_snapshot_time(snapshot_buckets)
 
-      @log.info "Most recent snapshot: #{most_recent_dates[0]}"
+      @log.info "Most recent snapshot: #{most_recent_date}"
 
+      # Find snapshots for the specified date
       snapshot_mapping = { }
       AWS.memoize do
         snapshot_buckets.keys.each do |index|
           found = false
           snapshot_buckets[index].each do |snapshot|
-            if snapshot.tags["time"] == most_recent_dates[0]
+            if snapshot.tags["time"] == most_recent_date
               snapshot_mapping[index] = snapshot
               found = true
               break

data/lib/ec2launcher/hostname_generator.rb

@@ -4,9 +4,13 @@
 require 'rubygems'
 require 'aws-sdk'
 
+require 'ec2launcher/backoff_runner'
+
 module EC2Launcher
   # Helper class to generate sequential, numbered host names
   class HostnameGenerator
+    include BackoffRunner
+
     # 
     # @param [AWS::EC2] ec2 EC2 object used to query for existing instances
     # @param [EC2Launcher::Environment] environment Environment to use for generating host names
@@ -70,7 +74,10 @@ module EC2Launcher
     def load_instances(prefix, suffix)
       @server_name_cache = []
       AWS.memoize do
-        server_instances = @ec2.instances.filter("tag:Name", "#{prefix}*#{suffix}*")
+        server_instances = nil
+        run_with_backoff(60, 1, "searching for instances") do
+          server_instances = @ec2.instances.filter("tag:Name", "#{prefix}*#{suffix}*")
+        end
         server_instances.each do |i|
           next if i.status == :terminated
           @server_name_cache << i.tags[:Name]

data/lib/ec2launcher/version.rb

@@ -2,5 +2,5 @@
 # Copyright (c) 2012 Sean Laurent
 #
 module EC2Launcher
-  VERSION = "1.3.10"
+  VERSION = "1.3.12"
 end

data/test/ec2launcher/dsl/config_parser_test.rb

@@ -0,0 +1,39 @@
+require_relative "../../test_helper"
+
+class ConfigParserTest < MiniTest::Unit::TestCase
+  def test_full_config_parses_correctly
+    sample_config_erb = %q{
+config do
+  environments "environments"
+  applications "applications"
+
+  package_manager "yum"
+  config_manager "chef"
+end
+}.gsub(/^ /, '')
+  
+    config_dsl = EC2Launcher::DSL::ConfigDSL.execute(sample_config_erb)
+    refute config_dsl.nil?
+    refute config_dsl.config.nil?
+
+    config = config_dsl.config
+    assert_equal "yum", config.package_manager
+    assert_equal "chef", config.config_manager
+
+    assert_kind_of Array, config.applications
+    assert_kind_of Array, config.environments
+
+    assert_equal "applications", config.applications[0]
+    assert_equal "environments", config.environments[0]
+  end
+
+  def test_empty_config_parses_correctly
+    sample_config_erb = %q{
+config do
+end
+}.gsub(/^ /, '')
+    config_dsl = EC2Launcher::DSL::ConfigDSL.execute(sample_config_erb)
+    refute config_dsl.nil?
+    refute config_dsl.config.nil?
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'minitest/autorun'
+require 'minitest/pride'
+require File.expand_path('../../lib/ec2launcher.rb', __FILE__)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ec2launcher
 version: !ruby/object:Gem::Version
-  version: 1.3.10
+  version: 1.3.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-30 00:00:00.000000000 Z
+date: 2012-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -43,6 +43,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Tool to manage application configurations and launch new EC2 instances
   based on the configurations.
 email: 
@@ -85,6 +117,8 @@ files:
 - startup-scripts/runurl
 - startup-scripts/setup.rb
 - startup-scripts/setup_instance.rb
+- test/ec2launcher/dsl/config_parser_test.rb
+- test/test_helper.rb
 - yard_extensions/dsl_attribute_handler.rb
 homepage: https://github.com/StudyBlue/ec2launcher
 licenses: []
@@ -110,5 +144,7 @@ rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Tool to launch EC2 instances.
-test_files: []
+test_files:
+- test/ec2launcher/dsl/config_parser_test.rb
+- test/test_helper.rb
 has_rdoc: