stemcell 0.12.2 → 0.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 42199bc80153e575d437ce9c618b662e6861af3e
-  data.tar.gz: 53e93477347e08429b03e74b4580d4cc714c424d
+SHA256:
+  metadata.gz: 440002a8d4c58fd1329664656e61589e6d174e7330acfa73af957e8d2e690d7c
+  data.tar.gz: 88a3fe213bfc823772d72296a130b37fca10032c8519597277d377e1abdfd798
 SHA512:
-  metadata.gz: 67d1742a36a3f003a849651d9f7b899414a738ce2ea2a684efaf77bee1399fc65d824c174a28abdf981c2aa739e137d973589d29b2a15ef8e94dfcc94e9f1b3d
-  data.tar.gz: 0e916fa0c134eb26ff01cba67c6b1e90ae921fa932940f3596a5d09b0607a52abae2b99099d3569e3cf4b67ac4e2a91226666e32af9a250af0a33f081bbd4f71
+  metadata.gz: 80f1c8e2e9f3fee310ff779dfcb9b1b4e7118ed27da2045441747b0783060fb9dc39022a6c23740167f76c9d4655999015a6858d68cdc2c906c73bfb2af99d4e
+  data.tar.gz: 07ba99d048effdccd90f9a1185449e13e9829cb047d2c16c027c91834accd2610b053b460bc0f5eba8983ca49312a3238d817d9d5946cd81bd57ff4ce021f4b3

data/.travis.yml

@@ -5,5 +5,6 @@ rvm:
   - 2.1.2
   - 2.2.4
   - 2.3.1
+  - 2.6.8
 before_install:
   - gem install bundler -v 1.16.3

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# 0.13.1
+- Support for specifying --min-count and --max-count
+
+# 0.13.1
+- Support for specifying --cpu-options
+
+# 0.13.0
+- Migrate to AWS SDK to v3
+- Drop support for ClassicLink
+- Removed `necrosis` script
+
 # 0.12.2
 - Support for using a custom EC2 endpoint
 

data/README.md

@@ -77,10 +77,10 @@ $ stemcell $your_chef_role --tail
 
 ### Terminating:
 
-To terminate, use the necrosis command and pass a space separated list of instance ids:
+To terminate, use the AWS CLI and pass a space separated list of instance ids:
 
 ```bash
-$ necrosis i-12345678 i-12345679 i-12345670
+$ aws ec2 terminate-instances --instance-ids i-12345678 i-12345679 i-12345670
 ```
 
 ## Automation ##

data/lib/stemcell/command_line.rb

@@ -132,6 +132,7 @@ module Stemcell
       # comma-separated list when presented as defaults.
       pd['security_groups'] &&= pd['security_groups'].join(',')
       pd['tags'] &&= pd['tags'].to_a.map { |p| p.join('=') }.join(',')
+      pd['cpu_options'] &&= pd['cpu_options'].to_a.map { |p| p.join('=') }.join(',')
       pd['chef_cookbook_attributes'] &&= pd['chef_cookbook_attributes'].join(',')
     end
 

data/lib/stemcell/launcher.rb

@@ -1,4 +1,5 @@
-require 'aws-sdk-v1'
+require 'aws-sdk-ec2'
+require 'base64'
 require 'logger'
 require 'erb'
 require 'set'
@@ -30,6 +31,7 @@ module Stemcell
       'chef_environment',
       'chef_data_bag_secret',
       'chef_data_bag_secret_path',
+      'cpu_options',
       'git_branch',
       'git_key',
       'git_origin',
@@ -45,10 +47,11 @@ module Stemcell
       'dedicated_tenancy',
       'associate_public_ip_address',
       'count',
+      'min_count',
+      'max_count',
       'security_groups',
       'security_group_ids',
       'tags',
-      'classic_link',
       'iam_role',
       'ebs_optimized',
       'termination_protection',
@@ -90,7 +93,7 @@ module Stemcell
       opts['git_key'] = try_file(opts['git_key'])
       opts['chef_data_bag_secret'] = try_file(opts['chef_data_bag_secret'])
 
-      # generate tags and merge in any that were specefied as inputs
+      # generate tags and merge in any that were specified as inputs
       tags = {
         'Name' => "#{opts['chef_role']}-#{opts['chef_environment']}",
         'Group' => "#{opts['chef_role']}-#{opts['chef_environment']}",
@@ -101,36 +104,46 @@ module Stemcell
       tags['Name'] = opts['chef_role'] if opts['chef_environment'] == 'production'
       tags.merge!(opts['tags']) if opts['tags']
 
+      #  Min/max number of instances to launch
+      min_count = opts['min_count'] || opts['count']
+      max_count = opts['max_count'] || opts['count']
+
       # generate launch options
       launch_options = {
         :image_id => opts['image_id'],
         :instance_type => opts['instance_type'],
         :key_name => opts['key_name'],
-        :count => opts['count'],
+        :min_count => min_count,
+        :max_count => max_count,
+      }
+
+      # Associate Public IP can only bet set on network_interfaces, and if present
+      # security groups and subnet should be set on the interface. VPC-only.
+      # Primary network interface
+      network_interface = {
+        device_index: 0,
       }
+      launch_options[:network_interfaces] = [network_interface]
 
       if opts['security_group_ids'] && !opts['security_group_ids'].empty?
-        launch_options[:security_group_ids] = opts['security_group_ids']
+        network_interface[:groups] = opts['security_group_ids']
       end
 
       if opts['security_groups'] && !opts['security_groups'].empty?
-        if @vpc_id
-          # convert sg names to sg ids as VPC only accepts ids
-          security_group_ids = get_vpc_security_group_ids(@vpc_id, opts['security_groups'])
-          launch_options[:security_group_ids] ||= []
-          launch_options[:security_group_ids].concat(security_group_ids)
-        else
-          launch_options[:security_groups] = opts['security_groups']
-        end
+        # convert sg names to sg ids as VPC only accepts ids
+        security_group_ids = get_vpc_security_group_ids(@vpc_id, opts['security_groups'])
+        network_interface[:groups] ||= []
+        network_interface[:groups].concat(security_group_ids)
       end
 
+      launch_options[:placement] = placement = {}
       # specify availability zone (optional)
       if opts['availability_zone']
-        launch_options[:availability_zone] = opts['availability_zone']
+        placement[:availability_zone] = opts['availability_zone']
       end
 
       if opts['subnet']
-        launch_options[:subnet] = opts['subnet']
+        network_interface[:subnet_id] = opts['subnet']
       end
 
       if opts['private_ip_address']
@@ -138,23 +151,23 @@ module Stemcell
       end
 
       if opts['dedicated_tenancy']
-        launch_options[:dedicated_tenancy] = opts['dedicated_tenancy']
+        placement[:tenancy] = 'dedicated'
       end
 
       if opts['associate_public_ip_address']
-        launch_options[:associate_public_ip_address] = opts['associate_public_ip_address']
+        network_interface[:associate_public_ip_address] = opts['associate_public_ip_address']
       end
 
       # specify IAM role (optional)
       if opts['iam_role']
-        launch_options[:iam_instance_profile] = opts['iam_role']
+        launch_options[:iam_instance_profile] = {
+          name: opts['iam_role']
+        }
       end
 
       # specify placement group (optional)
       if opts['placement_group']
-        launch_options[:placement] = {
-          :group_name => opts['placement_group'],
-        }
+        placement[:group_name] = opts['placement_group']
       end
 
       # specify an EBS-optimized instance (optional)
@@ -171,6 +184,11 @@ module Stemcell
         launch_options[:block_device_mappings] = opts['block_device_mappings']
       end
 
+      # specify cpu options (optional)
+      if opts['cpu_options']
+        launch_options[:cpu_options] = opts['cpu_options']
+      end
+
       # specify ephemeral block device mappings (optional)
       if opts['ephemeral_devices']
         launch_options[:block_device_mappings] ||= []
@@ -182,35 +200,31 @@ module Stemcell
         end
       end
 
+      if opts['termination_protection']
+        launch_options[:disable_api_termination] = true
+      end
+
       # generate user data script to bootstrap instance, include in launch
       # options UNLESS we have manually set the user-data (ie. for ec2admin)
-      launch_options[:user_data] = opts.fetch('user_data', render_template(opts))
+      launch_options[:user_data] = Base64.encode64(opts.fetch('user_data', render_template(opts)))
+
+      # add tags to launch options so we don't need to make a separate CreateTags call
+      launch_options[:tag_specifications] = [{
+        resource_type: 'instance',
+        tags: tags.map { |k, v| { key: k, value: v } }
+      }]
 
       # launch instances
       instances = do_launch(launch_options)
 
       # everything from here on out must succeed, or we kill the instances we just launched
       begin
-        # set tags on all instances launched
-        set_tags(instances, tags)
-        @log.info "sent ec2 api tag requests successfully"
-
-        # link to classiclink
-        unless @vpc_id
-          set_classic_link(instances, opts['classic_link'])
-          @log.info "successfully applied classic link settings (if any)"
-        end
-
-        # turn on termination protection
-        # we do this now to make sure all other settings worked
-        if opts['termination_protection']
-          enable_termination_protection(instances)
-          @log.info "successfully enabled termination protection"
-        end
-
         # wait for aws to report instance stats
         if opts.fetch('wait', true)
-          wait(instances)
+          instance_ids = instances.map(&:instance_id)
+          @log.info "Waiting up to #{MAX_RUNNING_STATE_WAIT_TIME} seconds for #{instances.count} " \
+                "instance(s): (#{instance_ids})"
+          instances = wait(instance_ids)
           print_run_info(instances)
           @log.info "launched instances successfully"
         end
@@ -227,19 +241,18 @@ module Stemcell
       return instances
     end
 
-    def kill(instances, opts={})
-      return if !instances || instances.empty?
+    def kill(instance_ids, opts={})
+      return if !instance_ids || instance_ids.empty?
 
-      errors = run_batch_operation(instances) do |instance|
-        begin
-          @log.warn "Terminating instance #{instance.id}"
-          instance.terminate
-          nil # nil == success
-        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          opts[:ignore_not_found] ? nil : e
-        end
-      end
-      check_errors(:kill, instances.map(&:id), errors)
+      @log.warn "Terminating instances #{instance_ids}"
+      ec2.terminate_instances(instance_ids: instance_ids)
+      nil # nil == success
+    rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
+      raise unless opts[:ignore_not_found]
+
+      invalid_ids = e.message.scan(/i-[a-z0-9]+/)
+      instance_ids -= invalid_ids
+      retry unless instance_ids.empty? || invalid_ids.empty? # don't retry if we couldn't find any instance ids
     end
 
     # this is made public for ec2admin usage
@@ -249,7 +262,7 @@ module Stemcell
       erb_template = ERB.new(template_file)
       last_bootstrap_line = LAST_BOOTSTRAP_LINE
       generated_template = erb_template.result(binding)
-      @log.debug "genereated template is #{generated_template}"
+      @log.debug "generated template is #{generated_template}"
       return generated_template
     end
 
@@ -266,16 +279,16 @@ module Stemcell
       puts "install logs will be in /var/log/init and /var/log/init.err"
     end
 
-    def wait(instances)
-      @log.info "Waiting up to #{MAX_RUNNING_STATE_WAIT_TIME} seconds for #{instances.count} " \
-                "instance(s): (#{instances.inspect})"
-
-      times_out_at = Time.now + MAX_RUNNING_STATE_WAIT_TIME
-      until instances.all?{ |i| i.status == :running }
-        wait_time_expire_or_sleep(times_out_at)
+    def wait(instance_ids)
+      started_at = Time.now
+      result = ec2.wait_until(:instance_running, instance_ids: instance_ids) do |w|
+        w.max_attempts = nil
+        w.delay = RUNNING_STATE_WAIT_SLEEP_TIME
+        w.before_wait do |attempts, response|
+          throw :failure if Time.now - started_at > MAX_RUNNING_STATE_WAIT_TIME
+        end
       end
-
-      @log.info "all instances in running state"
+      result.map { |page| page.reservations.map(&:instances) }.flatten
     end
 
     def verify_required_options(params, required_options)
@@ -291,35 +304,22 @@ module Stemcell
     def do_launch(opts={})
       @log.debug "about to launch instance(s) with options #{opts}"
       @log.info "launching instances"
-      instances = ec2.instances.create(opts)
-      instances = [instances] unless Array === instances
+      instances = ec2.run_instances(opts).instances
       instances.each do |instance|
         @log.info "launched instance #{instance.instance_id}"
       end
       return instances
     end
 
-    def set_tags(instances=[], tags)
-      @log.info "setting tags on instance(s)"
-      errors = run_batch_operation(instances) do |instance|
-        begin
-          instance.tags.set(tags)
-          nil # nil == success
-        rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
-          e
-        end
-      end
-      check_errors(:set_tags, instances.map(&:id), errors)
-    end
-
     # Resolve security group names to their ids in the given VPC
     def get_vpc_security_group_ids(vpc_id, group_names)
       group_map = {}
       @log.info "resolving security groups #{group_names} in #{vpc_id}"
-      vpc = AWS::EC2::VPC.new(vpc_id)
-      vpc.security_groups.each do |sg|
-        next if sg.vpc_id != vpc_id
-        group_map[sg.name] = sg.group_id
+      ec2.describe_security_groups(filters: [{ name: 'vpc-id', values: [vpc_id] }]).
+        each do |response|
+        response.security_groups.each do |sg|
+          group_map[sg.group_name] = sg.group_id
+        end
       end
       group_ids = []
       group_names.each do |sg_name|
@@ -329,135 +329,21 @@ module Stemcell
       group_ids
     end
 
-    def set_classic_link(left_to_process, classic_link)
-      return unless classic_link
-      return unless classic_link['vpc_id']
-
-      security_group_ids = classic_link['security_group_ids'] || []
-      security_group_names = classic_link['security_groups'] || []
-      return if security_group_ids.empty? && security_group_names.empty?
-
-      if !security_group_names.empty?
-        extra_group_ids = get_vpc_security_group_ids(classic_link['vpc_id'], security_group_names)
-        security_group_ids = security_group_ids + extra_group_ids
-      end
-
-      @log.info "applying classic link settings on #{left_to_process.count} instance(s)"
-
-      errors = []
-      processed = []
-      times_out_at = Time.now + MAX_RUNNING_STATE_WAIT_TIME
-      until left_to_process.empty?
-        wait_time_expire_or_sleep(times_out_at)
-
-        # we can only apply classic link when instances are in the running state
-        # lets apply classiclink as instances become available so we don't wait longer than necessary
-        recently_running = left_to_process.select{ |inst| inst.status == :running }
-        left_to_process = left_to_process.reject{ |inst| recently_running.include?(inst) }
-
-        processed += recently_running
-        errors += run_batch_operation(recently_running) do |instance|
-          begin
-            result = ec2.client.attach_classic_link_vpc({
-                :instance_id => instance.id,
-                :vpc_id => classic_link['vpc_id'],
-                :groups => security_group_ids,
-              })
-            result.error
-          rescue StandardError => e
-            e
-          end
-        end
-      end
-
-      check_errors(:set_classic_link, processed.map(&:id), errors)
-    end
-
-    def enable_termination_protection(instances)
-      @log.info "enabling termination protection on instance(s)"
-      errors = run_batch_operation(instances) do |instance|
-        begin
-          resp = ec2.client.modify_instance_attribute({
-              :instance_id => instance.id,
-              :disable_api_termination => {
-                :value => true
-              }
-            })
-          resp.error  # returns nil (success) unless there was an error
-        rescue StandardError => e
-          e
-        end
-      end
-      check_errors(:enable_termination_protection, instances.map(&:id), errors)
-    end
-
     # attempt to accept keys as file paths
     def try_file(opt="")
         File.read(File.expand_path(opt)) rescue opt
     end
 
-    INITIAL_RETRY_SEC = 1
-
-    # Return a Hash of instance => error. Empty hash indicates "no error"
-    # for code block:
-    #   - if block returns nil, success
-    #   - if block returns non-nil value (e.g., exception), retry 3 times w/ backoff
-    #   - if block raises exception, fail
-    def run_batch_operation(instances)
-      instances.map do |instance|
-        begin
-          attempt = 0
-          result = nil
-          while attempt < @max_attempts
-            # sleep idempotently except for the first attempt
-            sleep(INITIAL_RETRY_SEC * 2 ** attempt) if attempt != 0
-            result = yield(instance)
-            break if result.nil? # nil indicates success
-            attempt += 1
-          end
-          result # result for this instance is nil or returned exception
-        rescue => e
-          e # result for this instance is caught exception
-        end
-      end
-    end
-
-    def check_errors(operation, instance_ids, errors)
-      return if errors.all?(&:nil?)
-      raise IncompleteOperation.new(
-        operation,
-        instance_ids,
-        instance_ids.zip(errors).reject { |i, e| e.nil? }
-      )
-    end
-
     def ec2
-      return @ec2 if @ec2
-
-      if @vpc_id
-        @ec2 = AWS::EC2::VPC.new(@vpc_id)
-      else
-        @ec2 = AWS::EC2.new
-      end
-
-      @ec2
-    end
-
-    def wait_time_expire_or_sleep(times_out_at)
-      now = Time.now
-      if now >= times_out_at
-        raise TimeoutError, "exceded timeout of #{MAX_RUNNING_STATE_WAIT_TIME} seconds"
-      else
-        sleep [RUNNING_STATE_WAIT_SLEEP_TIME, times_out_at - now].min
+      @ec2 ||= begin
+        opts = @ec2_endpoint ? { endpoint: @ec2_endpoint } : {}
+        Aws::EC2::Client.new(opts)
       end
     end
 
     def configure_aws_creds_and_region
       # configure AWS with creds/region
       aws_configs = {:region => @region}
-      aws_configs.merge!({
-        :ec2_endpoint      => @ec2_endpoint
-      }) if @ec2_endpoint
       aws_configs.merge!({
         :access_key_id     => @aws_access_key,
         :secret_access_key => @aws_secret_key
@@ -465,7 +351,7 @@ module Stemcell
       aws_configs.merge!({
         :session_token     => @aws_session_token,
       }) if @aws_session_token
-      AWS.config(aws_configs)
+      Aws.config.update(aws_configs)
     end
   end
 end

data/lib/stemcell/option_parser.rb

@@ -96,24 +96,6 @@ module Stemcell
         :type  => String,
         :env   => 'VPC_ID'
       },
-      {
-        :name  => 'classic_link_vpc_id',
-        :desc  => 'VPC ID to which this instance will be classic-linked',
-        :type  => String,
-        :env   => 'CLASSIC_LINK_VPC_ID',
-      },
-      {
-        :name  => 'classic_link_security_group_ids',
-        :desc  => 'comma-separated list of security group IDs to link into ClassicLink; not used unless classic_link_vpc_id is set',
-        :type  => String,
-        :env   => 'CLASSIC_LINK_SECURITY_GROUP_IDS',
-      },
-      {
-        :name  => 'classic_link_security_groups',
-        :desc  => 'comma-separated list of security groups to link into ClassicLink; not used unless classic_link_vpc_id is set',
-        :type  => String,
-        :env   => 'CLASSIC_LINK_SECURITY_GROUPS',
-      },
       {
         :name  => 'subnet',
         :desc  => "VPC subnet for which to launch this instance",
@@ -205,6 +187,12 @@ module Stemcell
         :type  => String,
         :env   => 'CHEF_DATA_BAG_SECRET_PATH'
       },
+      {
+        :name => 'cpu_options',
+        :desc => "comma-separated list of cpu option key=value pairs",
+        :type => String,
+        :env => 'CPU_OPTIONS'
+      },
       {
         :name => 'chef_role',
         :desc => "chef role of instance to be launched",
@@ -271,6 +259,18 @@ module Stemcell
         :type  => Integer,
         :env   => 'COUNT'
       },
+      {
+        :name  => 'min_count',
+        :desc  => "minimum number of instances to launch",
+        :type  => Integer,
+        :env   => 'MIN_COUNT'
+      },
+      {
+        :name  => 'max_count',
+        :desc  => "maximum number of instances to launch",
+        :type  => Integer,
+        :env   => 'MAX_COUNT'
+      },
       {
         :name  => 'tail',
         :desc  => "interactively tail the initial converge",
@@ -363,6 +363,16 @@ module Stemcell
         options['tags'] = tags
       end
 
+      # convert cpu_options from comma separated string to ruby hash
+      if options['cpu_options']
+        cpu_options = {}
+        options['cpu_options'].split(',').each do |cpu_opt|
+          key, value = cpu_opt.split('=')
+          cpu_options[key] = value
+        end
+        options['cpu_options'] = cpu_options
+      end
+
       # parse block_device_mappings to convert it from the standard CLI format
       # to the EC2 Ruby API format.
       # All of this is a bit hard to find so here are some docs links to
@@ -430,17 +440,6 @@ module Stemcell
       # convert chef_cookbook_attributes from comma separated string to ruby array
       options['chef_cookbook_attributes'] &&= options['chef_cookbook_attributes'].split(',')
 
-      # format the classic link options
-      if options['classic_link_vpc_id']
-        options['classic_link']['vpc_id'] = options['classic_link_vpc_id']
-      end
-      if options['classic_link_security_group_ids']
-        options['classic_link']['security_group_ids'] = options['classic_link_security_group_ids'].split(',')
-      end
-      if options['classic_link_security_groups']
-        options['classic_link']['security_groups'] = options['classic_link_security_groups'].split(',')
-      end
-
       options
     end
 

data/lib/stemcell/version.rb

@@ -1,3 +1,3 @@
 module Stemcell
-  VERSION = "0.12.2"
+  VERSION = "0.13.2"
 end

data/spec/lib/stemcell/launcher_spec.rb

@@ -1,45 +1,56 @@
 require 'spec_helper'
+require 'base64'
 
-class MockInstance
-  def initialize(id)
-    @id = id
-  end
-
-  def id
-    @id
-  end
-
-  def status
-    :running
-  end
-end
-
-class MockSecurityGroup
-  attr_reader :group_id, :name, :vpc_id
-  def initialize(id, name, vpc_id)
-    @group_id = id
-    @name = name
-    @vpc_id = vpc_id
+describe Stemcell::Launcher do
+  before do
+    Aws.config[:stub_responses] = true
   end
-end
 
-class MockException < StandardError
-end
-
-describe Stemcell::Launcher do
   let(:launcher) {
     opts = {'region' => 'region'}
     launcher = Stemcell::Launcher.new(opts)
     launcher
   }
   let(:operation) { 'op' }
-  let(:instances) { (1..4).map { |id| MockInstance.new(id) } }
+  let(:instances) do
+    ('1'..'4').map do |id|
+      Aws::EC2::Types::Instance.new(
+        instance_id: id,
+        private_ip_address: "10.10.10.#{id}",
+        state: Aws::EC2::Types::InstanceState.new(name: 'pending')
+      )
+    end
+  end
   let(:instance_ids) { instances.map(&:id) }
 
   describe '#launch' do
-    let(:ec2) { instance_double(AWS::EC2) }
-    let(:client) { double(AWS::EC2::Client) }
-    let(:response) { instance_double(AWS::Core::Response) }
+    let(:ec2) do
+      ec2 = Aws::EC2::Client.new
+      ec2.stub_responses(
+        :describe_security_groups,
+        security_groups: [
+          {group_id: 'sg-1', group_name: 'sg_name1', vpc_id:'vpc-1'},
+          {group_id: 'sg-2', group_name: 'sg_name2', vpc_id:'vpc-1'},
+        ],
+      )
+      ec2.stub_responses(
+        :describe_instances,
+        reservations: [{
+          instances: ('1'..'4').map do |id|
+            {
+              instance_id: id,
+              private_ip_address: "10.10.10.#{id}",
+              public_ip_address: "24.10.10.#{id}",
+              state: {
+                name: 'running'
+              }
+            }
+          end
+        }]
+      )
+      ec2
+    end
+    let(:response) { instance_double(Seahorse::Client::Response) }
     let(:launcher) {
       opts = {'region' => 'region', 'vpc_id' => 'vpc-1'}
       launcher = Stemcell::Launcher.new(opts)
@@ -59,7 +70,9 @@ describe Stemcell::Launcher do
         'availability_zone'       => 'us-east-1a',
         'count'                   => 2,
         'security_groups'         => ['sg_name1', 'sg_name2'],
-        'wait'                    => false
+        'user'                    => 'some_user',
+        'wait'                    => true,
+        'cpu_options'             => 'core_count=1,threads_per_core=1'
       }
     }
 
@@ -67,146 +80,119 @@ describe Stemcell::Launcher do
       allow(launcher).to receive(:try_file).and_return('secret')
       allow(launcher).to receive(:render_template).and_return('template')
       allow(launcher).to receive(:ec2).and_return(ec2)
-      allow(ec2).to receive(:client).and_return(client)
       allow(response).to receive(:error).and_return(nil)
     end
 
     it 'launches all of the instances' do
       expect(launcher).to receive(:get_vpc_security_group_ids).
         with('vpc-1', ['sg_name1', 'sg_name2']).and_call_original
-      expect_any_instance_of(AWS::EC2::VPC).to receive(:security_groups).
-        and_return([1,2].map { |i| MockSecurityGroup.new("sg-#{i}", "sg_name#{i}", 'vpc-1')})
+      expect(ec2).to receive(:describe_security_groups).and_call_original
       expect(launcher).to receive(:do_launch).with(a_hash_including(
           :image_id           => 'ami-d9d6a6b0',
           :instance_type      => 'c1.xlarge',
           :key_name           => 'key',
-          :count              => 2,
-          :security_group_ids => ['sg-1', 'sg-2'],
-          :availability_zone  => 'us-east-1a',
-          :user_data          => 'template'
+          :min_count          => 2,
+          :max_count          => 2,
+          :placement          => { :availability_zone => 'us-east-1a' },
+          :network_interfaces => [{
+            :device_index => 0,
+            :groups => ['sg-1', 'sg-2' ]
+          }],
+          :tag_specifications => [
+            {
+              :resource_type => 'instance',
+              :tags => [
+                { :key => "Name",       :value => "role-environment" },
+                { :key => "Group",      :value => "role-environment" },
+                { :key => "created_by", :value => "some_user" },
+                { :key => "stemcell",   :value => Stemcell::VERSION },
+              ]},
+          ],
+          :user_data          => Base64.encode64('template'),
+          :cpu_options        => 'core_count=1,threads_per_core=1'
         )).and_return(instances)
-      expect(launcher).to receive(:set_tags).with(kind_of(Array), kind_of(Hash)).and_return(nil)
-      # set_classic_link should not be set on vpc hosts.
-      expect(launcher).not_to receive(:set_classic_link)
-
-      launcher.send(:launch, launch_options)
+      launched_instances = launcher.send(:launch, launch_options)
+      expect(launched_instances.map(&:public_ip_address)).to all(be_truthy)
     end
 
-    it 'calls set_classic_link for non vpc instances' do
-      launcher = Stemcell::Launcher.new({'region' => 'region', 'vpc_id' => false})
-      expect(launcher).to receive(:set_classic_link)
-      expect(launcher).to receive(:set_tags).with(kind_of(Array), kind_of(Hash)).and_return(nil)
-      expect(launcher).to receive(:do_launch).and_return(instances)
-      launcher.send(:launch, launch_options)
+    it 'launches with min/max count' do
+      expect(launcher).to receive(:do_launch).with(a_hash_including(
+        :min_count => 3,
+        :max_count => 6,
+      )).and_return(instances)
+      options = launch_options.merge(
+        'min_count' => 3,
+        'max_count' => 6,
+      )
+      launched_instances = launcher.send(:launch,
+        launch_options.merge('min_count' => 3, 'max_count' => 6))
     end
   end
 
-  describe '#set_classic_link' do
-    let(:ec2) { instance_double(AWS::EC2) }
-    let(:client) { double(AWS::EC2::Client) }
-    let(:response) { instance_double(AWS::Core::Response) }
-    before do
-      allow(launcher).to receive(:ec2).and_return(ec2)
-      allow(ec2).to receive(:client).and_return(client)
-      allow(response).to receive(:error).and_return(nil)
+  describe '#kill' do
+    let(:ec2) do
+      ec2 = Aws::EC2::Client.new
+      ec2.stub_responses(
+        :terminate_instances, -> (context) {
+        instance_ids = context.params[:instance_ids]
+        if instance_ids.include? 'i-3'
+          Aws::EC2::Errors::InvalidInstanceIDNotFound.new(nil, "The instance ID 'i-3' do not exist")
+        else
+          {} # success
+        end
+      })
+      ec2
     end
 
-    let(:classic_link) {
-      {
-        'vpc_id' => 'vpc-1',
-        'security_group_ids' => ['sg-1', 'sg-2'],
-        'security_groups' => ['sg_name']
-      }
-    }
-
-    it 'invokes classic link on all of the instances' do
-      expect(launcher).to receive(:get_vpc_security_group_ids).with('vpc-1', ['sg_name']).
-        and_call_original
-      expect_any_instance_of(AWS::EC2::VPC).to receive(:security_groups).
-        and_return([MockSecurityGroup.new('sg-3', 'sg_name', 'vpc-1')])
-      instances.each do |instance|
-        expect(client).to receive(:attach_classic_link_vpc).ordered.with(a_hash_including(
-            :instance_id => instance.id,
-            :vpc_id => classic_link['vpc_id'],
-            :groups => ['sg-1', 'sg-2', 'sg-3'],
-          )).and_return(response)
-      end
+    let(:instance_ids) { ('i-1'..'i-4').to_a }
 
-      launcher.send(:set_classic_link, instances, classic_link)
-    end
-  end
-
-  describe '#run_batch_operation' do
-    it "raises no exception when no internal error occur" do
-      errors = launcher.send(:run_batch_operation, instances) {}
-      expect(errors.all?(&:nil?)).to be true
+    before do
+      allow(launcher).to receive(:ec2).and_return(ec2)
     end
 
-    it "runs full batch even when there are two error" do
-      errors = launcher.send(:run_batch_operation,
-                             instances) do |instance, error|
-        raise "error-#{instance.id}" if instance.id % 2 == 0
+    context 'when ignore_not_found is true' do
+      it 'terminates valid instances even if an invalid instance id is provided' do
+        launcher.kill(instance_ids, ignore_not_found: true)
       end
-      expect(errors.count(&:nil?)).to be_eql(2)
-      expect(errors.reject(&:nil?).map { |e| e.message }).to \
-        be_eql([2, 4].map { |id| "error-#{id}" })
-    end
 
-    it "retries after an intermittent error" do
-      count = 0
-      errors = launcher.send(:run_batch_operation,
-                             instances)  do |instance|
-        if instance.id == 3
-          count += 1
-          count < 3 ?
-            AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error-#{instance.id}"):
-            nil
-        end
+      it 'finishes without error even if no instance ids are valid' do
+        launcher.kill(['i-3'], ignore_not_found: true)
       end
-      expect(errors.all?(&:nil?)).to be true
     end
 
-    it "retries up to max_attempts option per instance" do
-      max_attempts = 6
-      opts = {'region' => 'region', 'max_attempts' => max_attempts}
-      launcher = Stemcell::Launcher.new(opts)
-      allow(launcher).to receive(:sleep).and_return(0)
-      tags = double("Tags")
-      instances = (1..2).map do |id|
-        inst = MockInstance.new(id)
-        allow(inst).to receive(:tags).and_return(tags)
-        inst
+    context 'when ignore_not_found is false' do
+      it 'raises an error' do
+        expect { launcher.kill(instance_ids) }.to raise_error(Aws::EC2::Errors::InvalidInstanceIDNotFound)
       end
-      expect(tags).to receive(:set).with({'a' => 'b'}).exactly(12).times.
-        and_raise(AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error"))
-      expect do
-        launcher.send(:set_tags, instances, {'a' => 'b'})
-      end.to raise_error(Stemcell::IncompleteOperation)
     end
   end
 
   describe '#configure_aws_creds_and_region' do
-    it 'AWS region is configured after launcher is instanciated' do
-      expect(AWS.config.region).to be_eql('region')
+    it 'AWS region is configured after launcher is instantiated' do
+      expect(Aws.config[:region]).to be_eql('region')
     end
 
     it 'AWS region configuration changed' do
       mock_launcher = Stemcell::Launcher.new('region' => 'ap-northeast-1')
-      expect(AWS.config.region).to be_eql('ap-northeast-1')
+      expect(Aws.config[:region]).to be_eql('ap-northeast-1')
     end
   end
 
   describe '#ec2' do
+
     it 'can return a client with regional endpoint' do
-      launcher = Stemcell::Launcher.new({'region' => 'region1', 'ec2_endpoint' => nil})
+      launcher = Stemcell::Launcher.new({'region' => 'us-east-1', 'ec2_endpoint' => nil})
       client = launcher.send(:ec2)
-      expect(client.config.ec2_endpoint).to be_eql('ec2.region1.amazonaws.com')
+      expect(client.config[:endpoint].to_s).to be_eql('https://ec2.us-east-1.amazonaws.com')
     end
 
     it 'can return a client with custom endpoint' do
-      launcher = Stemcell::Launcher.new({'region' => 'region1', 'ec2_endpoint' => 'endpoint1'})
+      launcher = Stemcell::Launcher.new({
+        'region' => 'region1',
+        'ec2_endpoint' => 'https://endpoint1',
+      })
       client = launcher.send(:ec2)
-      expect(client.config.ec2_endpoint).to be_eql('endpoint1')
+      expect(client.config[:endpoint].to_s).to be_eql('https://endpoint1')
     end
   end
 end

data/stemcell.gemspec

@@ -17,8 +17,11 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'aws-sdk-v1', '~> 1.63'
-  s.add_runtime_dependency 'net-ssh',    '~> 2.9'
+  # pins several aws sdk transitive dependencies to maintain compatibility with Ruby < 2.3
+  s.add_runtime_dependency 'aws-eventstream', '~> 1.1.1'
+  s.add_runtime_dependency 'aws-sdk-ec2',     '~> 1'
+  s.add_runtime_dependency 'aws-sigv4',       '~> 1.2.4'
+  s.add_runtime_dependency 'net-ssh',         '~> 2.9'
   if RUBY_VERSION >= '2.0'
     s.add_runtime_dependency 'chef',     '>= 11.4.0'
   else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stemcell
 version: !ruby/object:Gem::Version
-  version: 0.12.2
+  version: 0.13.2
 platform: ruby
 authors:
 - Martin Rhoads
@@ -11,22 +11,50 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-18 00:00:00.000000000 Z
+date: 2022-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-v1
+  name: aws-eventstream
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.63'
+        version: 1.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.63'
+        version: 1.1.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ec2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sigv4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.4
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -157,7 +185,6 @@ description: A tool for launching and bootstrapping EC2 instances
 email:
 - igor.serebryany@airbnb.com
 executables:
-- necrosis
 - stemcell
 extensions: []
 extra_rdoc_files: []
@@ -169,7 +196,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/necrosis
 - bin/stemcell
 - examples/stemcell.json
 - examples/stemcellrc
@@ -241,8 +267,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: no summary

data/bin/necrosis

@@ -1,114 +0,0 @@
-#!/usr/bin/env ruby
-
-# -*- mode: shell -*-
-
-require 'aws-sdk-v1'
-require 'trollop'
-
-options = Trollop::options do
-  version "Necrosis 0.1.0 (c) Airbnb, Inc."
-  banner  "Necrosis: the killing script"
-
-  opt('aws_access_key',
-      "aws access key",
-      :type => String,
-      :default => ENV['AWS_ACCESS_KEY']
-      )
-
-  opt('aws_secret_key',
-      "aws secret key",
-      :type => String,
-      :default => ENV['AWS_SECRET_KEY']
-      )
-
-  opt('aws_regions',
-      "comma-separated list of aws regions to search",
-      :type => String,
-      :default => ENV['AWS_REGIONS'],
-      :short => :r
-      )
-
-  opt :non_interactive, 'Do not ask confirmation', :short => :f
-end
-
-NON_INTERACTIVE = options[:non_interactive]
-
-required_parameters = %w(aws_access_key aws_secret_key)
-
-required_parameters.each do |arg|
-  raise ArgumentError, "--#{arg.gsub('_','-')} needs to be specified on the commandline or set \
-by the #{arg.upcase.gsub('-','_')} environment variable" if
-    options[arg].nil? or ! options[arg]
-end
-
-raise ArgumentError, "you did not provide any instance ids to kill" if ARGV.empty?
-
-# a hash from instance_id => [ec2 instance objects]
-instances = ARGV.inject({}) { |h, n| h[n] = []; h }
-
-# convert comma-separated list to Array
-regions = (options['aws_regions'] || '').split(',')
-
-def authorized? instance
-  return true if NON_INTERACTIVE
-
-  puts 'Terminate? (y/N)'
-  confirm = $stdin.gets
-  confirmed = confirm && confirm.chomp.downcase == 'y'
-
-  if confirmed
-    age = Time.now - instance.launch_time
-    days = age / (24*3600)
-    if days > 2
-      puts "Instance is #{days} days old. REALLY terminate? (y/N)"
-      confirm = $stdin.gets
-      confirmed = confirm && confirm.chomp.downcase == 'y'
-    end
-  end
-
-  return confirmed
-end
-
-def delete_instance id, instance, region
-  if instance.api_termination_disabled?
-    puts "Cannot terminate instance #{id} -- termination protection enabled"
-    return
-  end
-
-  puts "Instance #{id} (#{instance.status} in #{region.name})"
-  puts "\tKey name: #{instance.key_name}"
-  puts "\tLaunched: #{instance.launch_time}"
-  instance.tags.to_h.each do |k, v|
-    puts "\t#{k} : #{v}"
-  end
-
-  if authorized? instance
-    instance.terminate
-    puts "Instance #{id} terminated"
-  end
-end
-
-AWS.memoize do
-  ec2 = AWS::EC2.new(:access_key_id => options['aws_access_key'], :secret_access_key => options['aws_secret_key'])
-  ec2.regions.each do |region|
-    next unless regions.empty? || regions.include?(region.name)
-    instances.each do |id, objects|
-      instance = region.instances[id]
-      objects << [instance, region] if instance.exists?
-    end
-  end
-
-  instances.each do |id, objects|
-    case objects.count
-    when 0
-      STDERR.puts "No instance #{id} found"
-      next
-    when 1
-      instance, region = objects.first
-      delete_instance id, instance, region
-    else
-      puts "Found multiple instances named #{id}"
-      next
-    end
-  end
-end