stemcell 0.12.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d0a73c0c28344f526396892ffde6de0054696d7f
-  data.tar.gz: 3286ce5584f761e5e643f9fe02bd575f9199e651
+SHA256:
+  metadata.gz: 9942211d9334ce09b4a56299f2bc7e3e903d57eb871a60a8a1228c549e6b1407
+  data.tar.gz: 3e70bbdc1434124922c4be3cfb29e6842d6cda8210a1555fe3c36e4887740a11
 SHA512:
-  metadata.gz: 00ddd6ab1066baf755627d945f0fc033457f9256f891176a5af5b1b0d259fc93885d08a1d5b12cce4cbe42733be0b8af36ae6003fab521d7535cd00dd9da57e8
-  data.tar.gz: 4e09e64bba35df73ec82622889863d3456b162a6d9405954f02bb0087709bbf9a0036d052d543823871ed712d0c6171222a2bddbd7ab9e94eb9bdf7f351a036b
+  metadata.gz: c4f0a32c8fe858b23bd3ddbbb30ca752a0b23c0a2782d9fd0cf56d9cda0e344bdaa3c2361b1df9f4d5f5413ca88e15c714c868ee2efaa1b304dd74d248601fdb
+  data.tar.gz: a9fb4751d0fe25b94f4c47785da168a9e0e909464d0354461adc8db5d01487d5ea696ebcbc79e74267b6ccd114a794a5e5d13ed90bd6b831a20c3bb5ae5b4906

data/.travis.yml

@@ -5,5 +5,6 @@ rvm:
   - 2.1.2
   - 2.2.4
   - 2.3.1
+  - 2.6.8
 before_install:
-  - gem install bundler
+  - gem install bundler -v 1.16.3

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+# 0.13.1
+- Support for specifying --cpu-options
+
+# 0.13.0
+- Migrate to AWS SDK to v3
+- Drop support for ClassicLink
+- Removed `necrosis` script
+
+# 0.12.2
+- Support for using a custom EC2 endpoint
+
+# 0.12.1
+- Add support for Amazon Linux to the default bootstrap script
+- Allow setting backing_store options per region
+- Display private ip for launched instances
+
 # 0.12.0
 - Require Nokogiri ~> 1.8.2 due to vulnerability CVE-2017-15412
 - Require ruby version >= 2.1 for Nokogiri compatibility

data/README.md

@@ -77,10 +77,10 @@ $ stemcell $your_chef_role --tail
 
 ### Terminating:
 
-To terminate, use the necrosis command and pass a space separated list of instance ids:
+To terminate, use the AWS CLI and pass a space separated list of instance ids:
 
 ```bash
-$ necrosis i-12345678 i-12345679 i-12345670
+$ aws ec2 terminate-instances --instance-ids i-12345678 i-12345679 i-12345670
 ```
 
 ## Automation ##

data/examples/stemcell.json

@@ -9,10 +9,14 @@
 
   "backing_store": {
     "ebs": {
-      "image_id": "ami-23d9a94a"
+      "us-east-1": {
+        "image_id": "ami-23d9a94a"
+      }
     },
     "instance_store": {
-      "image_id": "ami-d9d6a6b0"
+      "us-east-1": {
+        "image_id": "ami-d9d6a6b0"
+      }
     }
   },
 

data/lib/stemcell/command_line.rb

@@ -132,6 +132,7 @@ module Stemcell
       # comma-separated list when presented as defaults.
       pd['security_groups'] &&= pd['security_groups'].join(',')
       pd['tags'] &&= pd['tags'].to_a.map { |p| p.join('=') }.join(',')
+      pd['cpu_options'] &&= pd['cpu_options'].to_a.map { |p| p.join('=') }.join(',')
       pd['chef_cookbook_attributes'] &&= pd['chef_cookbook_attributes'].join(',')
     end
 

data/lib/stemcell/launcher.rb

@@ -1,4 +1,5 @@
-require 'aws-sdk-v1'
+require 'aws-sdk-ec2'
+require 'base64'
 require 'logger'
 require 'erb'
 require 'set'
@@ -30,6 +31,7 @@ module Stemcell
       'chef_environment',
       'chef_data_bag_secret',
       'chef_data_bag_secret_path',
+      'cpu_options',
       'git_branch',
       'git_key',
       'git_origin',
@@ -48,7 +50,6 @@ module Stemcell
       'security_groups',
       'security_group_ids',
       'tags',
-      'classic_link',
       'iam_role',
       'ebs_optimized',
       'termination_protection',
@@ -75,6 +76,7 @@ module Stemcell
 
       @region = opts['region']
       @vpc_id = opts['vpc_id']
+      @ec2_endpoint = opts['ec2_endpoint']
       @aws_access_key = opts['aws_access_key']
       @aws_secret_key = opts['aws_secret_key']
       @aws_session_token = opts['aws_session_token']
@@ -89,7 +91,7 @@ module Stemcell
       opts['git_key'] = try_file(opts['git_key'])
       opts['chef_data_bag_secret'] = try_file(opts['chef_data_bag_secret'])
 
-      # generate tags and merge in any that were specefied as inputs
+      # generate tags and merge in any that were specified as inputs
       tags = {
         'Name' => "#{opts['chef_role']}-#{opts['chef_environment']}",
         'Group' => "#{opts['chef_role']}-#{opts['chef_environment']}",
@@ -105,31 +107,38 @@ module Stemcell
         :image_id => opts['image_id'],
         :instance_type => opts['instance_type'],
         :key_name => opts['key_name'],
-        :count => opts['count'],
+        :min_count => opts['count'],
+        :max_count => opts['count'],
       }
 
+
+      # Associate Public IP can only bet set on network_interfaces, and if present
+      # security groups and subnet should be set on the interface. VPC-only.
+      # Primary network interface
+      network_interface = {
+        device_index: 0,
+      }
+      launch_options[:network_interfaces] = [network_interface]
+
       if opts['security_group_ids'] && !opts['security_group_ids'].empty?
-        launch_options[:security_group_ids] = opts['security_group_ids']
+        network_interface[:groups] = opts['security_group_ids']
       end
 
       if opts['security_groups'] && !opts['security_groups'].empty?
-        if @vpc_id
-          # convert sg names to sg ids as VPC only accepts ids
-          security_group_ids = get_vpc_security_group_ids(@vpc_id, opts['security_groups'])
-          launch_options[:security_group_ids] ||= []
-          launch_options[:security_group_ids].concat(security_group_ids)
-        else
-          launch_options[:security_groups] = opts['security_groups']
-        end
+        # convert sg names to sg ids as VPC only accepts ids
+        security_group_ids = get_vpc_security_group_ids(@vpc_id, opts['security_groups'])
+        network_interface[:groups] ||= []
+        network_interface[:groups].concat(security_group_ids)
       end
 
+      launch_options[:placement] = placement = {}
       # specify availability zone (optional)
       if opts['availability_zone']
-        launch_options[:availability_zone] = opts['availability_zone']
+        placement[:availability_zone] = opts['availability_zone']
       end
 
       if opts['subnet']
-        launch_options[:subnet] = opts['subnet']
+        network_interface[:subnet_id] = opts['subnet']
       end
 
       if opts['private_ip_address']
@@ -137,23 +146,23 @@ module Stemcell
       end
 
       if opts['dedicated_tenancy']
-        launch_options[:dedicated_tenancy] = opts['dedicated_tenancy']
+        placement[:tenancy] = 'dedicated'
       end
 
       if opts['associate_public_ip_address']
-        launch_options[:associate_public_ip_address] = opts['associate_public_ip_address']
+        network_interface[:associate_public_ip_address] = opts['associate_public_ip_address']
       end
 
       # specify IAM role (optional)
       if opts['iam_role']
-        launch_options[:iam_instance_profile] = opts['iam_role']
+        launch_options[:iam_instance_profile] = {
+          name: opts['iam_role']
+        }
       end
 
       # specify placement group (optional)
       if opts['placement_group']
-        launch_options[:placement] = {
-          :group_name => opts['placement_group'],
-        }
+        placement[:group_name] = opts['placement_group']
       end
 
       # specify an EBS-optimized instance (optional)
@@ -170,6 +179,11 @@ module Stemcell
         launch_options[:block_device_mappings] = opts['block_device_mappings']
       end
 
+      # specify cpu options (optional)
+      if opts['cpu_options']
+        launch_options[:cpu_options] = opts['cpu_options']
+      end
+
       # specify ephemeral block device mappings (optional)
       if opts['ephemeral_devices']
         launch_options[:block_device_mappings] ||= []
@@ -181,35 +195,31 @@ module Stemcell
         end
       end
 
+      if opts['termination_protection']
+        launch_options[:disable_api_termination] = true
+      end
+
       # generate user data script to bootstrap instance, include in launch
       # options UNLESS we have manually set the user-data (ie. for ec2admin)
-      launch_options[:user_data] = opts.fetch('user_data', render_template(opts))
+      launch_options[:user_data] = Base64.encode64(opts.fetch('user_data', render_template(opts)))
+
+      # add tags to launch options so we don't need to make a separate CreateTags call
+      launch_options[:tag_specifications] = [{
+        resource_type: 'instance',
+        tags: tags.map { |k, v| { key: k, value: v } }
+      }]
 
       # launch instances
       instances = do_launch(launch_options)
 
       # everything from here on out must succeed, or we kill the instances we just launched
       begin
-        # set tags on all instances launched
-        set_tags(instances, tags)
-        @log.info "sent ec2 api tag requests successfully"
-
-        # link to classiclink
-        unless @vpc_id
-          set_classic_link(instances, opts['classic_link'])
-          @log.info "successfully applied classic link settings (if any)"
-        end
-
-        # turn on termination protection
-        # we do this now to make sure all other settings worked
-        if opts['termination_protection']
-          enable_termination_protection(instances)
-          @log.info "successfully enabled termination protection"
-        end
-
         # wait for aws to report instance stats
         if opts.fetch('wait', true)
-          wait(instances)
+          instance_ids = instances.map(&:instance_id)
+          @log.info "Waiting up to #{MAX_RUNNING_STATE_WAIT_TIME} seconds for #{instances.count} " \
+                "instance(s): (#{instance_ids})"
+          instances = wait(instance_ids)
           print_run_info(instances)
           @log.info "launched instances successfully"
         end
@@ -226,19 +236,18 @@ module Stemcell
       return instances
     end
 
-    def kill(instances, opts={})
-      return if !instances || instances.empty?
+    def kill(instance_ids, opts={})
+      return if !instance_ids || instance_ids.empty?
 
-      errors = run_batch_operation(instances) do |instance|
-        begin
-          @log.warn "Terminating instance #{instance.id}"
-          instance.terminate
-          nil # nil == success
-        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          opts[:ignore_not_found] ? nil : e
-        end
-      end
-      check_errors(:kill, instances.map(&:id), errors)
+      @log.warn "Terminating instances #{instance_ids}"
+      ec2.terminate_instances(instance_ids: instance_ids)
+      nil # nil == success
+    rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
+      raise unless opts[:ignore_not_found]
+
+      invalid_ids = e.message.scan(/i-[a-z0-9]+/)
+      instance_ids -= invalid_ids
+      retry unless instance_ids.empty? || invalid_ids.empty? # don't retry if we couldn't find any instance ids
     end
 
     # this is made public for ec2admin usage
@@ -248,7 +257,7 @@ module Stemcell
       erb_template = ERB.new(template_file)
       last_bootstrap_line = LAST_BOOTSTRAP_LINE
       generated_template = erb_template.result(binding)
-      @log.debug "genereated template is #{generated_template}"
+      @log.debug "generated template is #{generated_template}"
       return generated_template
     end
 
@@ -258,22 +267,23 @@ module Stemcell
       puts "\nhere is the info for what's launched:"
       instances.each do |instance|
         puts "\tinstance_id: #{instance.instance_id}"
-        puts "\tpublic ip:   #{instance.public_ip_address}"
+        puts "\tpublic ip:   #{instance.public_ip_address || 'none'}"
+        puts "\tprivate ip:  #{instance.private_ip_address || 'none'}"
         puts
       end
       puts "install logs will be in /var/log/init and /var/log/init.err"
     end
 
-    def wait(instances)
-      @log.info "Waiting up to #{MAX_RUNNING_STATE_WAIT_TIME} seconds for #{instances.count} " \
-                "instance(s): (#{instances.inspect})"
-
-      times_out_at = Time.now + MAX_RUNNING_STATE_WAIT_TIME
-      until instances.all?{ |i| i.status == :running }
-        wait_time_expire_or_sleep(times_out_at)
+    def wait(instance_ids)
+      started_at = Time.now
+      result = ec2.wait_until(:instance_running, instance_ids: instance_ids) do |w|
+        w.max_attempts = nil
+        w.delay = RUNNING_STATE_WAIT_SLEEP_TIME
+        w.before_wait do |attempts, response|
+          throw :failure if Time.now - started_at > MAX_RUNNING_STATE_WAIT_TIME
+        end
       end
-
-      @log.info "all instances in running state"
+      result.map { |page| page.reservations.map(&:instances) }.flatten
     end
 
     def verify_required_options(params, required_options)
@@ -289,35 +299,22 @@ module Stemcell
     def do_launch(opts={})
       @log.debug "about to launch instance(s) with options #{opts}"
       @log.info "launching instances"
-      instances = ec2.instances.create(opts)
-      instances = [instances] unless Array === instances
+      instances = ec2.run_instances(opts).instances
       instances.each do |instance|
         @log.info "launched instance #{instance.instance_id}"
       end
       return instances
     end
 
-    def set_tags(instances=[], tags)
-      @log.info "setting tags on instance(s)"
-      errors = run_batch_operation(instances) do |instance|
-        begin
-          instance.tags.set(tags)
-          nil # nil == success
-        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          e
-        end
-      end
-      check_errors(:set_tags, instances.map(&:id), errors)
-    end
-
     # Resolve security group names to their ids in the given VPC
     def get_vpc_security_group_ids(vpc_id, group_names)
       group_map = {}
       @log.info "resolving security groups #{group_names} in #{vpc_id}"
-      vpc = AWS::EC2::VPC.new(vpc_id, :ec2_endpoint => "ec2.#{@region}.amazonaws.com")
-      vpc.security_groups.each do |sg|
-        next if sg.vpc_id != vpc_id
-        group_map[sg.name] = sg.group_id
+      ec2.describe_security_groups(filters: [{ name: 'vpc-id', values: [vpc_id] }]).
+        each do |response|
+        response.security_groups.each do |sg|
+          group_map[sg.group_name] = sg.group_id
+        end
       end
       group_ids = []
       group_names.each do |sg_name|
@@ -327,129 +324,15 @@ module Stemcell
       group_ids
     end
 
-    def set_classic_link(left_to_process, classic_link)
-      return unless classic_link
-      return unless classic_link['vpc_id']
-
-      security_group_ids = classic_link['security_group_ids'] || []
-      security_group_names = classic_link['security_groups'] || []
-      return if security_group_ids.empty? && security_group_names.empty?
-
-      if !security_group_names.empty?
-        extra_group_ids = get_vpc_security_group_ids(classic_link['vpc_id'], security_group_names)
-        security_group_ids = security_group_ids + extra_group_ids
-      end
-
-      @log.info "applying classic link settings on #{left_to_process.count} instance(s)"
-
-      errors = []
-      processed = []
-      times_out_at = Time.now + MAX_RUNNING_STATE_WAIT_TIME
-      until left_to_process.empty?
-        wait_time_expire_or_sleep(times_out_at)
-
-        # we can only apply classic link when instances are in the running state
-        # lets apply classiclink as instances become available so we don't wait longer than necessary
-        recently_running = left_to_process.select{ |inst| inst.status == :running }
-        left_to_process = left_to_process.reject{ |inst| recently_running.include?(inst) }
-
-        processed += recently_running
-        errors += run_batch_operation(recently_running) do |instance|
-          begin
-            result = ec2.client.attach_classic_link_vpc({
-                :instance_id => instance.id,
-                :vpc_id => classic_link['vpc_id'],
-                :groups => security_group_ids,
-              })
-            result.error
-          rescue StandardError => e
-            e
-          end
-        end
-      end
-
-      check_errors(:set_classic_link, processed.map(&:id), errors)
-    end
-
-    def enable_termination_protection(instances)
-      @log.info "enabling termination protection on instance(s)"
-      errors = run_batch_operation(instances) do |instance|
-        begin
-          resp = ec2.client.modify_instance_attribute({
-              :instance_id => instance.id,
-              :disable_api_termination => {
-                :value => true
-              }
-            })
-          resp.error  # returns nil (success) unless there was an error
-        rescue StandardError => e
-          e
-        end
-      end
-      check_errors(:enable_termination_protection, instances.map(&:id), errors)
-    end
-
     # attempt to accept keys as file paths
     def try_file(opt="")
         File.read(File.expand_path(opt)) rescue opt
     end
 
-    INITIAL_RETRY_SEC = 1
-
-    # Return a Hash of instance => error. Empty hash indicates "no error"
-    # for code block:
-    #   - if block returns nil, success
-    #   - if block returns non-nil value (e.g., exception), retry 3 times w/ backoff
-    #   - if block raises exception, fail
-    def run_batch_operation(instances)
-      instances.map do |instance|
-        begin
-          attempt = 0
-          result = nil
-          while attempt < @max_attempts
-            # sleep idempotently except for the first attempt
-            sleep(INITIAL_RETRY_SEC * 2 ** attempt) if attempt != 0
-            result = yield(instance)
-            break if result.nil? # nil indicates success
-            attempt += 1
-          end
-          result # result for this instance is nil or returned exception
-        rescue => e
-          e # result for this instance is caught exception
-        end
-      end
-    end
-
-    def check_errors(operation, instance_ids, errors)
-      return if errors.all?(&:nil?)
-      raise IncompleteOperation.new(
-        operation,
-        instance_ids,
-        instance_ids.zip(errors).reject { |i, e| e.nil? }
-      )
-    end
-
     def ec2
-      return @ec2 if @ec2
-
-      # calculate our ec2 url
-      ec2_url = "ec2.#{@region}.amazonaws.com"
-
-      if @vpc_id
-        @ec2 = AWS::EC2::VPC.new(@vpc_id, :ec2_endpoint => ec2_url)
-      else
-        @ec2 = AWS::EC2.new(:ec2_endpoint => ec2_url)
-      end
-
-      @ec2
-    end
-
-    def wait_time_expire_or_sleep(times_out_at)
-      now = Time.now
-      if now >= times_out_at
-        raise TimeoutError, "exceded timeout of #{MAX_RUNNING_STATE_WAIT_TIME} seconds"
-      else
-        sleep [RUNNING_STATE_WAIT_SLEEP_TIME, times_out_at - now].min
+      @ec2 ||= begin
+        opts = @ec2_endpoint ? { endpoint: @ec2_endpoint } : {}
+        Aws::EC2::Client.new(opts)
       end
     end
 
@@ -463,7 +346,7 @@ module Stemcell
       aws_configs.merge!({
         :session_token     => @aws_session_token,
       }) if @aws_session_token
-      AWS.config(aws_configs)
+      Aws.config.update(aws_configs)
     end
   end
 end

data/lib/stemcell/metadata_launcher.rb

@@ -101,6 +101,7 @@ module Stemcell
         'region'            => options['region'],
         'vpc_id'            => options['vpc_id'],
         'max_attempts'      => options['batch_operation_retries'],
+        'ec2_endpoint'      => options['ec2_endpoint'],
       })
       # Slice off just the options used for launching.
       launch_options = {}

data/lib/stemcell/metadata_source/configuration.rb

@@ -21,10 +21,10 @@ module Stemcell
         validate_configutation
       end
 
-      def options_for_backing_store(backing_store)
+      def options_for_backing_store(backing_store, region)
         options = backing_store_options[backing_store]
         raise UnknownBackingStoreError.new(backing_store) if options.nil?
-        options
+        options.fetch(region, options)
       end
 
       def random_az_for_region(region)

data/lib/stemcell/metadata_source.rb

@@ -84,9 +84,14 @@ module Stemcell
       backing_store ||= config.default_options['backing_store']
       backing_store ||= DEFAULT_BACKING_STORE
 
+      backing_store_region   = override_options['region']
+      backing_store_region ||= role_options.to_hash['region'] if role_options
+      backing_store_region ||= config.default_options['region']
+      backing_store_region ||= DEFAULT_OPTIONS['region']
+
       # Step 3: Retrieve the backing store options from the defaults.
 
-      backing_store_options = config.options_for_backing_store(backing_store)
+      backing_store_options = config.options_for_backing_store(backing_store, backing_store_region)
       backing_store_options['backing_store'] = backing_store
 
       # Step 4: Merge the options together in priority order.

data/lib/stemcell/option_parser.rb

@@ -42,6 +42,12 @@ module Stemcell
         :env   => 'AWS_SESSION_TOKEN',
         :hide  => true
       },
+      {
+        :name  => 'ec2_endpoint',
+        :desc  => 'EC2 endpoint',
+        :type  => String,
+        :env   => 'EC2_ENDPOINT',
+      },
       {
         :name  => 'region',
         :desc  => "ec2 region to launch in",
@@ -90,24 +96,6 @@ module Stemcell
         :type  => String,
         :env   => 'VPC_ID'
       },
-      {
-        :name  => 'classic_link_vpc_id',
-        :desc  => 'VPC ID to which this instance will be classic-linked',
-        :type  => String,
-        :env   => 'CLASSIC_LINK_VPC_ID',
-      },
-      {
-        :name  => 'classic_link_security_group_ids',
-        :desc  => 'comma-separated list of security group IDs to link into ClassicLink; not used unless classic_link_vpc_id is set',
-        :type  => String,
-        :env   => 'CLASSIC_LINK_SECURITY_GROUP_IDS',
-      },
-      {
-        :name  => 'classic_link_security_groups',
-        :desc  => 'comma-separated list of security groups to link into ClassicLink; not used unless classic_link_vpc_id is set',
-        :type  => String,
-        :env   => 'CLASSIC_LINK_SECURITY_GROUPS',
-      },
       {
         :name  => 'subnet',
         :desc  => "VPC subnet for which to launch this instance",
@@ -199,6 +187,12 @@ module Stemcell
         :type  => String,
         :env   => 'CHEF_DATA_BAG_SECRET_PATH'
       },
+      {
+        :name => 'cpu_options',
+        :desc => "comma-separated list of cpu option key=value pairs",
+        :type => String,
+        :env => 'CPU_OPTIONS'
+      },
       {
         :name => 'chef_role',
         :desc => "chef role of instance to be launched",
@@ -357,6 +351,16 @@ module Stemcell
         options['tags'] = tags
       end
 
+      # convert cpu_options from comma separated string to ruby hash
+      if options['cpu_options']
+        cpu_options = {}
+        options['cpu_options'].split(',').each do |cpu_opt|
+          key, value = cpu_opt.split('=')
+          cpu_options[key] = value
+        end
+        options['cpu_options'] = cpu_options
+      end
+
       # parse block_device_mappings to convert it from the standard CLI format
       # to the EC2 Ruby API format.
       # All of this is a bit hard to find so here are some docs links to
@@ -424,17 +428,6 @@ module Stemcell
       # convert chef_cookbook_attributes from comma separated string to ruby array
       options['chef_cookbook_attributes'] &&= options['chef_cookbook_attributes'].split(',')
 
-      # format the classic link options
-      if options['classic_link_vpc_id']
-        options['classic_link']['vpc_id'] = options['classic_link_vpc_id']
-      end
-      if options['classic_link_security_group_ids']
-        options['classic_link']['security_group_ids'] = options['classic_link_security_group_ids'].split(',')
-      end
-      if options['classic_link_security_groups']
-        options['classic_link']['security_groups'] = options['classic_link_security_groups'].split(',')
-      end
-
       options
     end