stemcell 0.8.0 → 0.8.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    Yjk5MmIxNTA2OTg1ZTI1YzVlOTc5NTM3ODFiMjlmNzNmNjdkMjJhMQ==
-  data.tar.gz: !binary |-
-    ZDVkYmQ1YzViMzJlNjhmZDBlNGM2YTVmYTZmNjFmM2U3N2VhYjdhZQ==
+SHA1:
+  metadata.gz: 72fabe0d6ead6f78ac3e45619a500e6896e0c6c9
+  data.tar.gz: b53a9302344e97be08143b8ea814d5f2e5a3f092
 SHA512:
-  metadata.gz: !binary |-
-    YTU1M2YxOGEzNGFmZjE5YmE4NjZkNzk3Mjg4OTIxMGFjZWIwOGI1MTBmYmY0
-    YjdkYWIzNTBkMzY3N2E1ZjE1ZTVlY2U3MjJjODJmNjg1ZGQxMGMyOTYwOGRk
-    ZGQ1OWI2ZDI1ZGY4OTM1MjkwZTlkMDhkOWIyZTE3OTU4NGU4ZTc=
-  data.tar.gz: !binary |-
-    MDYwMzllODg3OTYxN2I5NGE1MWE3ZWE4ODliN2NiMmJkMGM4YjBlZGQ2ZDVm
-    MTU0ZjQwZGQ0NjNjNTNhMWRjN2Y1ZDM2M2IyNDdkNWU4NDdmYjJhOGZjNWI5
-    YTk1MDYyZjUyZjVkOWJhMDBjMGQ5ODA4MTBlYmQxYTgwMTJhNGI=
+  metadata.gz: 06803bd29174493c9127e88575c7847d5782ac2fd77c26687c92d5eaaf3254f882abfc1948fecaa07ccaa77837183b6872bbbef6b3ed2eeda10ebce279f508ad
+  data.tar.gz: cc936abe49a4c3690bb38700c6d70877a6e68bf632a55d4cce732cca48a707b7524db8a546f3279391d53d68e1c532107285a975eedb6e302917915595bec305

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 # next release
 - ...
 
+# 0.8.1
+- Add retry mechanism for instances launch/termination
+- Make `Launcher::launch!` transaction-like, which reclaims all partially launched instances in the event of non-intermittent error
+- Display better error message with reason and failed instances
+- Take converge lock during initial converge
+
 # 0.8.0
 - Support for VPC [Brenden](https://github.com/brndnmtthws)
 - Support relative paths and home alias in `Launcher#try_file` [Patrick Viet](https://github.com/patrickviet)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    stemcell (0.8.0)
+    stemcell (0.8.1)
       aws-creds (~> 0.2.2)
       aws-sdk (~> 1.9)
       chef (>= 11.4.0)
@@ -15,7 +15,9 @@ GEM
   specs:
     aws-creds (0.2.3)
       trollop (~> 2.0)
-    aws-sdk (1.51.0)
+    aws-sdk (1.62.0)
+      aws-sdk-v1 (= 1.62.0)
+    aws-sdk-v1 (1.62.0)
       json (~> 1.4)
       nokogiri (>= 1.4.4)
     chef (11.14.6)
@@ -46,18 +48,18 @@ GEM
     diff-lcs (1.2.5)
     docile (1.1.5)
     erubis (2.7.0)
-    ffi (1.9.3)
-    ffi-yajl (1.0.2)
+    ffi (1.9.6)
+    ffi-yajl (1.3.1)
       ffi (~> 1.5)
-      libyajl2 (~> 1.0)
+      libyajl2 (~> 1.2)
     hashie (2.1.2)
     highline (1.6.21)
     ipaddress (0.8.0)
     json (1.7.7)
-    libyajl2 (1.0.1)
+    libyajl2 (1.2.0)
     method_source (0.8.2)
     mime-types (1.25.1)
-    mini_portile (0.6.0)
+    mini_portile (0.6.2)
     mixlib-authentication (1.3.0)
       mixlib-log
     mixlib-cli (1.5.0)
@@ -65,14 +67,14 @@ GEM
     mixlib-log (1.6.0)
     mixlib-shellout (1.4.0)
     multi_json (1.10.1)
-    net-ssh (2.9.1)
+    net-ssh (2.9.2)
     net-ssh-gateway (1.2.0)
       net-ssh (>= 2.6.5)
     net-ssh-multi (1.2.0)
       net-ssh (>= 2.6.5)
       net-ssh-gateway (>= 1.2.0)
-    nokogiri (1.6.3.1)
-      mini_portile (= 0.6.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
     ohai (7.2.4)
       ffi (~> 1.9)
       ffi-yajl (~> 1.0)
@@ -89,7 +91,7 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rack (1.5.2)
+    rack (1.6.0)
     rake (10.3.2)
     rest-client (1.6.7)
       mime-types (>= 1.16)
@@ -110,7 +112,7 @@ GEM
     simplecov-html (0.8.0)
     slop (3.6.0)
     systemu (2.6.4)
-    trollop (2.0)
+    trollop (2.1.1)
     wmi-lite (1.0.0)
 
 PLATFORMS

data/examples/stemcellrc

@@ -1,6 +1,5 @@
 # these options need to be set
 export AWS_ACCESS_KEY=your_aws_access_key
-export AWS_SECRET_KEY=your_aws_secret_key
 
 # AWC keys can be managed from the AWS console by going
 #   to EC2 and then clicking 'Key Pairs'. Every EC2 instance

data/lib/stemcell/errors.rb

@@ -23,4 +23,20 @@ module Stemcell
       @option = option
     end
   end
+
+  class IncompleteOperation < Error
+    attr_reader :operation, :all_instance_ids, :errors
+    def initialize(operation, all_instance_ids, errors)
+      super()
+      @operation = operation
+      @all_instance_ids = all_instance_ids
+      @errors = errors
+    end
+
+    def message
+      "Incomplete operation '#{@operation}': " +
+      "all_instance_ids=#{@all_instance_ids.join('|')}; " +
+      "errors=" + (@errors.map { |k, v| "'#{k}' => '#{v}'"}.join('|'))
+    end
+  end
 end

data/lib/stemcell/launcher.rb

@@ -1,6 +1,7 @@
 require 'aws-sdk'
 require 'logger'
 require 'erb'
+require 'set'
 
 require "stemcell/version"
 require "stemcell/option_parser"
@@ -176,8 +177,6 @@ module Stemcell
         end
       end
 
-      #
-
       # generate user data script to bootstrap instance, include in launch
       # options UNLESS we have manually set the user-data (ie. for ec2admin)
       launch_options[:user_data] = opts.fetch('user_data', render_template(opts))
@@ -186,14 +185,24 @@ module Stemcell
       instances = do_launch(launch_options)
 
       # set tags on all instances launched
-      set_tags(instances, tags)
-      @log.info "sent ec2 api requests successfully"
-
-      # wait for aws to report instance stats
-      if opts.fetch('wait', true)
-        wait(instances)
-        print_run_info(instances)
-        @log.info "launched instances successfully"
+      begin
+        set_tags(instances, tags)
+        @log.info "sent ec2 api requests successfully"
+
+        # wait for aws to report instance stats
+        if opts.fetch('wait', true)
+          wait(instances)
+          print_run_info(instances)
+          @log.info "launched instances successfully"
+        end
+      rescue => e
+        @log.info "launch failed, killing all launched instances"
+        begin
+          kill(instances, :ignore_not_found => true)
+        rescue => kill_error
+          @log.warn "encountered an error during cleanup: #{kill_error.message}"
+        end
+        raise e
       end
 
       return instances
@@ -203,17 +212,20 @@ module Stemcell
       return @ec2.instances[id]
     end
 
-    def kill(instances,opts={})
-      return if instances.nil?
-      instances.each do |i|
+    def kill(instance_ids, opts={})
+      return if instance_ids.nil?
+
+      errors = run_batch_operation(instance_ids) do |id|
         begin
-          instance = find_instance(i)
+          instance = find_instance(id)
           @log.warn "Terminating instance #{instance.instance_id}"
           instance.terminate
+          nil # nil == success
         rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          throw e unless opts[:ignore_not_found]
+          opts[:ignore_not_found] ? nil : e
         end
       end
+      check_errors(:kill, instance_ids, errors)
     end
 
     # this is made public for ec2admin usage
@@ -243,15 +255,12 @@ module Stemcell
       @log.info "Waiting up to #{@timeout} seconds for #{instances.count} " \
                 "instance(s) (#{instances.inspect}):"
 
-      while true
-        sleep 5
-        if Time.now - @start_time > @timeout
-          kill(instances)
+      while !instances.all? { |i| i.status == :running }
+        elapsed = Time.now - @start_time
+        if elapsed >= @timeout
           raise TimeoutError, "exceded timeout of #{@timeout}"
-        end
-
-        if instances.select{|i| i.status != :running }.empty?
-          break
+        else
+          sleep min(5, @timeout - elapsed)
         end
       end
 
@@ -279,11 +288,17 @@ module Stemcell
       return instances
     end
 
-    def set_tags(instances=[],tags)
+    def set_tags(instances=[], tags)
       @log.info "setting tags on instance(s)"
-      instances.each do |instance|
-        instance.tags.set(tags)
+      errors = run_batch_operation(instances) do |instance|
+        begin
+          instance.tags.set(tags)
+          nil # nil == success
+        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
+          e
+        end
       end
+      check_errors(:set_tags, instances.map(&:id), errors)
     end
 
     # attempt to accept keys as file paths
@@ -291,5 +306,40 @@ module Stemcell
         File.read(File.expand_path(opt)) rescue opt
     end
 
+    MAX_ATTEMPTS = 3
+    INITIAL_RETRY_SEC = 1
+
+    # Return a Hash of instance => error. Empty hash indicates "no error"
+    # for code block:
+    #   - if block returns nil, success
+    #   - if block returns non-nil value (e.g., exception), retry 3 times w/ backoff
+    #   - if block raises exception, fail
+    def run_batch_operation(instances)
+      instances.map do |instance|
+        begin
+          attempt = 0
+          result = nil
+          while attempt < MAX_ATTEMPTS
+            # sleep idempotently except for the first attempt
+            sleep(INITIAL_RETRY_SEC * 2 ** attempt) if attempt != 0
+            result = yield(instance)
+            break if result.nil? # nil indicates success
+            attempt += 1
+          end
+          result # result for this instance is nil or returned exception
+        rescue => e
+          e # result for this instance is caught exception
+        end
+      end
+    end
+
+    def check_errors(operation, instance_ids, errors)
+      return if errors.all?(&:nil?)
+      raise IncompleteOperation.new(
+        operation,
+        instance_ids,
+        instance_ids.zip(errors).reject { |i, e| e.nil? }
+      )
+    end
   end
 end

data/lib/stemcell/templates/bootstrap.sh.erb

@@ -8,6 +8,11 @@
 # Martin Rhoads
 
 
+(
+echo 'Acquiring converge lock...'
+/usr/bin/flock -e 200
+echo 'Lock acquired!'
+
 set -o pipefail
 PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
@@ -146,6 +151,7 @@ repo_dir =    "${repo_dir}"
 cookbook_path "#{repo_dir}/cookbooks"
 role_path     "#{repo_dir}/roles"
 data_bag_path "#{repo_dir}/data_bags"
+ssl_verify_mode :verify_peer
 
 log_level        :info
 log_location     STDOUT
@@ -252,3 +258,4 @@ configure_chef_daemon
 
 
 echo "<%= last_bootstrap_line %>"
+) 200> /var/run/converge.lock

data/lib/stemcell/version.rb

@@ -1,3 +1,3 @@
 module Stemcell
-  VERSION = "0.8.0"
+  VERSION = "0.8.1"
 end

data/spec/lib/stemcell/launcher_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+class MockInstance
+  def initialize(id)
+    @id = id
+  end
+
+  def id
+    @id
+  end
+end
+
+class MockException < StandardError
+end
+
+describe Stemcell::Launcher do
+  let(:launcher) {
+    opts = {}
+    Stemcell::Launcher::REQUIRED_OPTIONS.map { |k| opts[k] = "" }
+    launcher = Stemcell::Launcher.new(opts)
+    launcher
+  }
+  let(:operation) { 'op' }
+  let(:instances) { (1..4).map { |id| MockInstance.new(id) } }
+  let(:instance_ids) { instances.map(&:id) }
+
+  describe '#run_batch_operation' do
+
+    it "raises no exception when no internal error occur" do
+      errors = launcher.send(:run_batch_operation, instances) {}
+      errors.all?(&:nil?).should be_true
+    end
+
+    it "runs full batch even when there are two error" do
+      errors = launcher.send(:run_batch_operation,
+                             instances) do |instance, error|
+        raise "error-#{instance.id}" if instance.id % 2 == 0
+      end
+      errors.count(&:nil?).should be_eql(2)
+      errors.reject(&:nil?).map { |e| e.message }.should \
+        be_eql([2, 4].map { |id| "error-#{id}" })
+    end
+
+    it "retries after an intermittent error" do
+      count = 0
+      errors = launcher.send(:run_batch_operation,
+                             instances)  do |instance|
+        if instance.id == 3
+          count += 1
+          count < 3 ?
+            AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error-#{instance.id}"):
+            nil
+        end
+      end
+      errors.all?(&:nil?).should be_true
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stemcell
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Martin Rhoads
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-21 00:00:00.000000000 Z
+date: 2015-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -45,14 +45,14 @@ dependencies:
   name: chef
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 11.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 11.4.0
 - !ruby/object:Gem::Dependency
@@ -166,6 +166,7 @@ files:
 - spec/fixtures/chef_repo/stemcell-backing-store-missing.json
 - spec/fixtures/chef_repo/stemcell-defaults-missing.json
 - spec/fixtures/chef_repo/stemcell.json
+- spec/lib/stemcell/launcher_spec.rb
 - spec/lib/stemcell/metadata_source/chef_repository_spec.rb
 - spec/lib/stemcell/metadata_source/configuration_spec.rb
 - spec/lib/stemcell/metadata_source_spec.rb
@@ -182,17 +183,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: no summary
@@ -218,6 +219,7 @@ test_files:
 - spec/fixtures/chef_repo/stemcell-backing-store-missing.json
 - spec/fixtures/chef_repo/stemcell-defaults-missing.json
 - spec/fixtures/chef_repo/stemcell.json
+- spec/lib/stemcell/launcher_spec.rb
 - spec/lib/stemcell/metadata_source/chef_repository_spec.rb
 - spec/lib/stemcell/metadata_source/configuration_spec.rb
 - spec/lib/stemcell/metadata_source_spec.rb