stemcell 0.11.11 → 0.13.0

data/lib/stemcell/templates/bootstrap.sh.erb

@@ -54,16 +54,33 @@ encrypted_data_bag_secret_path='<%= opts['chef_data_bag_secret_path'].to_s %>'
 ##
 
 update() {
-  echo updating apt repo
-  apt-get update 1>&2
+  echo updating repository metadata
+
+  case $OS_PACKAGE_MANAGER in
+    apt)
+      apt-get update 1>&2
+      ;;
+    yum)
+      # yum install will check if metadata needs to be updated
+      # so it's just for consistency with apt-based distros
+      yum check-update 1>&2 || true
+      ;;
+  esac
 }
 
 install() {
-  if ! (dpkg -l | awk '{print $2}' | grep -q ^$1$ ) ; then
-    echo installing $1...
-    export DEBIAN_FRONTEND=noninteractive
-    apt-get install -y $1 1>&2
-  fi
+  case $OS_PACKAGE_MANAGER in
+    apt)
+      if ! (dpkg -l | awk '{print $2}' | grep -q ^$1$ ) ; then
+        echo installing $1...
+        export DEBIAN_FRONTEND=noninteractive
+        apt-get install -y $1 1>&2
+      fi
+      ;;
+    yum)
+      rpm -q $1 2>&1 >/dev/null || yum install -y $1
+      ;;
+  esac
 }
 
 set_hostname() {
@@ -86,22 +103,34 @@ set_hostname() {
   echo "$local_ip $fqdn $hostname" >> /etc/hosts
 }
 
+download_chef_package() {
+  echo "Downloading chef from $1"
+  wget "$1" -O "$2" --progress=dot:mega --tries=15 --retry-connrefused --timeout=5 --wait=10 --random-wait --continue --no-dns-cache
+}
+
 install_chef() {
+  package_local="/tmp/chef_${chef_version}.${OS_PACKAGE_EXT}"
+  package_url="$(render_chef_url)"
+
   # check to see if chef is already installed
   if ! which chef-solo > /dev/null ; then
-    platform=`lsb_release -s -i | tr '[:upper:]' '[:lower:]'`
-    platform_version=`lsb_release -s -r`
-    arch=`uname -m`
-    package_local="/tmp/chef_${chef_version}.deb"
-    package_url="<%= opts['chef_package_source'] %>"
-    echo "Downloading chef from $package_url"
-    wget $package_url -O $package_local --progress=dot:mega
-    echo "Installing chef $chef_version"
-    dpkg -i $package_local
-    rm $package_local
+    case $OS_PACKAGE_MANAGER in
+      apt)
+        download_chef_package "$package_url" "$package_local"
+        echo "Installing chef $chef_version"
+        dpkg -i $package_local
+        ;;
+      yum)
+        download_chef_package "$package_url" "$package_local"
+        echo "Installing chef $chef_version"
+        rpm -ivh "$package_local"
+        ;;
+    esac
   else
     echo chef is already installed
   fi
+
+  rm -f $package_local
 }
 
 create_ohai_hint() {
@@ -147,16 +176,17 @@ configure_chef() {
 
   echo "configuring chef solo..."
   cat<<EOF > ${chef_dir}/solo.rb
+# chef config (https://docs.chef.io/config_rb_client.html)
 repo_dir = "${repo_dir}"
 
 cookbook_path ["#{repo_dir}/site-cookbooks", "#{repo_dir}/cookbooks"]
 role_path "#{repo_dir}/roles"
 data_bag_path "#{repo_dir}/data_bags"
+ohai.plugin_path << "#{repo_dir}/ohai_plugins"
 ssl_verify_mode :verify_peer
 log_level :info
 log_location STDOUT
 
-Ohai::Config[:plugin_path] << "#{repo_dir}/ohai_plugins"
 EOF
 
   if [ -n "$encrypted_data_bag_secret_path" ]; then
@@ -197,7 +227,7 @@ git reset --hard origin/\${branch}
 git clean -fdx
 
 json="{\"role\": \"\${role}\", \"env\": \"\${env}\", \"branch\": \"\${branch}\"}"
-json_file=\`tempfile\`
+json_file=\`mktemp\`
 echo \$json > \$json_file
 trap "{ rm -f '\$json_file' ; }" EXIT
 
@@ -235,12 +265,55 @@ run_chef() {
   set -e
 }
 
+render_chef_url() {
+  base_url="<%= opts['chef_package_source'] %>"
+  echo $base_url | sed \
+    -e "s/%{platform}/$OS_PLATFORM/g" \
+    -e "s/%{platform_version}/$OS_PLATFORM_VERSION/g" \
+    -e "s/%{chef_version}/$chef_version/g" \
+    -e "s/%{package_manager}/$OS_PACKAGE_MANAGER/g" \
+    -e "s/%{package_arch}/$OS_PACKAGE_ARCH/g" \
+    -e "s/%{package_ext}/$OS_PACKAGE_EXT/g"
+}
+
+check_os_version() {
+  NAME=unsupported
+
+  if [ -f /etc/os-release ]; then
+    . /etc/os-release
+  fi
+
+  case "$NAME" in
+    "Ubuntu")
+      OS_VERSION="$NAME $VERSION_ID"
+      OS_PLATFORM=ubuntu
+      OS_PLATFORM_VERSION=$VERSION_ID
+      OS_PACKAGE_MANAGER=apt
+      OS_PACKAGE_ARCH=$(dpkg --print-architecture)
+      OS_PACKAGE_EXT=deb
+      ;;
+    "Amazon Linux")
+      OS_VERSION="$NAME $VERSION_ID"
+      OS_PLATFORM=amazon
+      OS_PLATFORM_VERSION=$VERSION_ID
+      OS_PACKAGE_MANAGER=yum
+      OS_PACKAGE_ARCH=$(uname -m)
+      OS_PACKAGE_EXT=rpm
+      ;;
+    *)
+      echo "Unsupported operating system"
+      exit 1
+      ;;
+  esac
+}
 
 ##
 ## main
 ##
 
-echo "starting chef bootstrapping..."
+check_os_version
+
+echo "starting chef bootstrapping on ${OS_VERSION}..."
 update
 install curl
 install git

data/lib/stemcell/version.rb

@@ -1,3 +1,3 @@
 module Stemcell
-  VERSION = "0.11.11"
+  VERSION = "0.13.0"
 end

data/spec/fixtures/chef_repo/stemcell-azs-missing.json

@@ -4,7 +4,9 @@
   },
   "backing_store": {
     "instance_store": {
-      "image_id": "ami-d9d6a6b0"
+      "us-east-1": {
+        "image_id": "ami-d9d6a6b0"
+      }
     }
   }
 }

data/spec/fixtures/chef_repo/stemcell-backing-store-legacy.json

@@ -0,0 +1,13 @@
+{
+  "defaults": {
+    "instance_type": "m1.small"
+  },
+  "backing_store": {
+    "instance_store": {
+      "image_id": "ami-d9d6a6b0"
+    }
+  },
+  "availability_zones": {
+    "us-east-1": ["us-east-1a"]
+  }
+}

data/spec/fixtures/chef_repo/stemcell-cookbook-attribute.json

@@ -5,7 +5,9 @@
   },
   "backing_store": {
     "instance_store": {
-      "image_id": "ami-d9d6a6b0"
+      "us-east-1": {
+        "image_id": "ami-d9d6a6b0"
+      }
     }
   },
   "availability_zones": {

data/spec/fixtures/chef_repo/stemcell-defaults-missing.json

@@ -1,7 +1,9 @@
 {
   "backing_store": {
     "instance_store": {
-      "baz": "woo"
+      "us-east-1": {
+        "baz": "woo"
+      }
     }
   },
   "availability_zones": {

data/spec/fixtures/chef_repo/stemcell-options-parser.json

@@ -16,10 +16,14 @@
 
   "backing_store": {
     "hvm1": {
-      "image_id": "ami-1"
+      "us-east-1": {
+        "image_id": "ami-1"
+      }
     },
     "pv1": {
-      "image_id": "ami-2"
+      "us-east-1": {
+        "image_id": "ami-2"
+      }
     }
   },
 

data/spec/fixtures/chef_repo/stemcell.json

@@ -4,7 +4,9 @@
   },
   "backing_store": {
     "instance_store": {
-      "image_id": "ami-d9d6a6b0"
+      "us-east-1": {
+        "image_id": "ami-d9d6a6b0"
+      }
     }
   },
   "availability_zones": {

data/spec/lib/stemcell/launcher_spec.rb

@@ -1,45 +1,56 @@
 require 'spec_helper'
+require 'base64'
 
-class MockInstance
-  def initialize(id)
-    @id = id
-  end
-
-  def id
-    @id
-  end
-
-  def status
-    :running
-  end
-end
-
-class MockSecurityGroup
-  attr_reader :group_id, :name, :vpc_id
-  def initialize(id, name, vpc_id)
-    @group_id = id
-    @name = name
-    @vpc_id = vpc_id
+describe Stemcell::Launcher do
+  before do
+    Aws.config[:stub_responses] = true
   end
-end
-
-class MockException < StandardError
-end
 
-describe Stemcell::Launcher do
   let(:launcher) {
     opts = {'region' => 'region'}
     launcher = Stemcell::Launcher.new(opts)
     launcher
   }
   let(:operation) { 'op' }
-  let(:instances) { (1..4).map { |id| MockInstance.new(id) } }
+  let(:instances) do
+    ('1'..'4').map do |id|
+      Aws::EC2::Types::Instance.new(
+        instance_id: id,
+        private_ip_address: "10.10.10.#{id}",
+        state: Aws::EC2::Types::InstanceState.new(name: 'pending')
+      )
+    end
+  end
   let(:instance_ids) { instances.map(&:id) }
 
   describe '#launch' do
-    let(:ec2) { instance_double(AWS::EC2) }
-    let(:client) { double(AWS::EC2::Client) }
-    let(:response) { instance_double(AWS::Core::Response) }
+    let(:ec2) do
+      ec2 = Aws::EC2::Client.new
+      ec2.stub_responses(
+        :describe_security_groups,
+        security_groups: [
+          {group_id: 'sg-1', group_name: 'sg_name1', vpc_id:'vpc-1'},
+          {group_id: 'sg-2', group_name: 'sg_name2', vpc_id:'vpc-1'},
+        ],
+      )
+      ec2.stub_responses(
+        :describe_instances,
+        reservations: [{
+          instances: ('1'..'4').map do |id|
+            {
+              instance_id: id,
+              private_ip_address: "10.10.10.#{id}",
+              public_ip_address: "24.10.10.#{id}",
+              state: {
+                name: 'running'
+              }
+            }
+          end
+        }]
+      )
+      ec2
+    end
+    let(:response) { instance_double(Seahorse::Client::Response) }
     let(:launcher) {
       opts = {'region' => 'region', 'vpc_id' => 'vpc-1'}
       launcher = Stemcell::Launcher.new(opts)
@@ -59,7 +70,8 @@ describe Stemcell::Launcher do
         'availability_zone'       => 'us-east-1a',
         'count'                   => 2,
         'security_groups'         => ['sg_name1', 'sg_name2'],
-        'wait'                    => false
+        'user'                    => 'some_user',
+        'wait'                    => true
       }
     }
 
@@ -67,132 +79,105 @@ describe Stemcell::Launcher do
       allow(launcher).to receive(:try_file).and_return('secret')
       allow(launcher).to receive(:render_template).and_return('template')
       allow(launcher).to receive(:ec2).and_return(ec2)
-      allow(ec2).to receive(:client).and_return(client)
       allow(response).to receive(:error).and_return(nil)
     end
 
     it 'launches all of the instances' do
       expect(launcher).to receive(:get_vpc_security_group_ids).
         with('vpc-1', ['sg_name1', 'sg_name2']).and_call_original
-      expect_any_instance_of(AWS::EC2::VPC).to receive(:security_groups).
-        and_return([1,2].map { |i| MockSecurityGroup.new("sg-#{i}", "sg_name#{i}", 'vpc-1')})
+      expect(ec2).to receive(:describe_security_groups).and_call_original
       expect(launcher).to receive(:do_launch).with(a_hash_including(
           :image_id           => 'ami-d9d6a6b0',
           :instance_type      => 'c1.xlarge',
           :key_name           => 'key',
-          :count              => 2,
-          :security_group_ids => ['sg-1', 'sg-2'],
-          :availability_zone  => 'us-east-1a',
-          :user_data          => 'template'
+          :min_count          => 2,
+          :max_count          => 2,
+          :placement          => { :availability_zone => 'us-east-1a' },
+          :network_interfaces => [{
+            :device_index => 0,
+            :groups => ['sg-1', 'sg-2' ]
+          }],
+          :tag_specifications => [
+            {
+              :resource_type => 'instance',
+              :tags => [
+                { :key => "Name",       :value => "role-environment" },
+                { :key => "Group",      :value => "role-environment" },
+                { :key => "created_by", :value => "some_user" },
+                { :key => "stemcell",   :value => Stemcell::VERSION },
+              ]},
+          ],
+          :user_data          => Base64.encode64('template')
         )).and_return(instances)
-      expect(launcher).to receive(:set_tags).with(kind_of(Array), kind_of(Hash)).and_return(nil)
-      # set_classic_link should not be set on vpc hosts.
-      expect(launcher).not_to receive(:set_classic_link)
-
-      launcher.send(:launch, launch_options)
-    end
-
-    it 'calls set_classic_link for non vpc instances' do
-      launcher = Stemcell::Launcher.new({'region' => 'region', 'vpc_id' => false})
-      expect(launcher).to receive(:set_classic_link)
-      expect(launcher).to receive(:set_tags).with(kind_of(Array), kind_of(Hash)).and_return(nil)
-      expect(launcher).to receive(:do_launch).and_return(instances)
-      launcher.send(:launch, launch_options)
+      launched_instances = launcher.send(:launch, launch_options)
+      expect(launched_instances.map(&:public_ip_address)).to all(be_truthy)
     end
   end
 
-  describe '#set_classic_link' do
-    let(:ec2) { instance_double(AWS::EC2) }
-    let(:client) { double(AWS::EC2::Client) }
-    let(:response) { instance_double(AWS::Core::Response) }
-    before do
-      allow(launcher).to receive(:ec2).and_return(ec2)
-      allow(ec2).to receive(:client).and_return(client)
-      allow(response).to receive(:error).and_return(nil)
+  describe '#kill' do
+    let(:ec2) do
+      ec2 = Aws::EC2::Client.new
+      ec2.stub_responses(
+        :terminate_instances, -> (context) {
+        instance_ids = context.params[:instance_ids]
+        if instance_ids.include? 'i-3'
+          Aws::EC2::Errors::InvalidInstanceIDNotFound.new(nil, "The instance ID 'i-3' do not exist")
+        else
+          {} # success
+        end
+      })
+      ec2
     end
 
-    let(:classic_link) {
-      {
-        'vpc_id' => 'vpc-1',
-        'security_group_ids' => ['sg-1', 'sg-2'],
-        'security_groups' => ['sg_name']
-      }
-    }
-
-    it 'invokes classic link on all of the instances' do
-      expect(launcher).to receive(:get_vpc_security_group_ids).with('vpc-1', ['sg_name']).
-        and_call_original
-      expect_any_instance_of(AWS::EC2::VPC).to receive(:security_groups).
-        and_return([MockSecurityGroup.new('sg-3', 'sg_name', 'vpc-1')])
-      instances.each do |instance|
-        expect(client).to receive(:attach_classic_link_vpc).ordered.with(a_hash_including(
-            :instance_id => instance.id,
-            :vpc_id => classic_link['vpc_id'],
-            :groups => ['sg-1', 'sg-2', 'sg-3'],
-          )).and_return(response)
-      end
+    let(:instance_ids) { ('i-1'..'i-4').to_a }
 
-      launcher.send(:set_classic_link, instances, classic_link)
-    end
-  end
-
-  describe '#run_batch_operation' do
-    it "raises no exception when no internal error occur" do
-      errors = launcher.send(:run_batch_operation, instances) {}
-      expect(errors.all?(&:nil?)).to be true
+    before do
+      allow(launcher).to receive(:ec2).and_return(ec2)
     end
 
-    it "runs full batch even when there are two error" do
-      errors = launcher.send(:run_batch_operation,
-                             instances) do |instance, error|
-        raise "error-#{instance.id}" if instance.id % 2 == 0
+    context 'when ignore_not_found is true' do
+      it 'terminates valid instances even if an invalid instance id is provided' do
+        launcher.kill(instance_ids, ignore_not_found: true)
       end
-      expect(errors.count(&:nil?)).to be_eql(2)
-      expect(errors.reject(&:nil?).map { |e| e.message }).to \
-        be_eql([2, 4].map { |id| "error-#{id}" })
-    end
 
-    it "retries after an intermittent error" do
-      count = 0
-      errors = launcher.send(:run_batch_operation,
-                             instances)  do |instance|
-        if instance.id == 3
-          count += 1
-          count < 3 ?
-            AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error-#{instance.id}"):
-            nil
-        end
+      it 'finishes without error even if no instance ids are valid' do
+        launcher.kill(['i-3'], ignore_not_found: true)
       end
-      expect(errors.all?(&:nil?)).to be true
     end
 
-    it "retries up to max_attempts option per instance" do
-      max_attempts = 6
-      opts = {'region' => 'region', 'max_attempts' => max_attempts}
-      launcher = Stemcell::Launcher.new(opts)
-      allow(launcher).to receive(:sleep).and_return(0)
-      tags = double("Tags")
-      instances = (1..2).map do |id|
-        inst = MockInstance.new(id)
-        allow(inst).to receive(:tags).and_return(tags)
-        inst
+    context 'when ignore_not_found is false' do
+      it 'raises an error' do
+        expect { launcher.kill(instance_ids) }.to raise_error(Aws::EC2::Errors::InvalidInstanceIDNotFound)
       end
-      expect(tags).to receive(:set).with({'a' => 'b'}).exactly(12).times.
-        and_raise(AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error"))
-      expect do
-        launcher.send(:set_tags, instances, {'a' => 'b'})
-      end.to raise_error(Stemcell::IncompleteOperation)
     end
   end
 
   describe '#configure_aws_creds_and_region' do
-    it 'AWS region is configured after launcher is instanciated' do
-      expect(AWS.config.region).to be_eql('region')
+    it 'AWS region is configured after launcher is instantiated' do
+      expect(Aws.config[:region]).to be_eql('region')
     end
 
     it 'AWS region configuration changed' do
       mock_launcher = Stemcell::Launcher.new('region' => 'ap-northeast-1')
-      expect(AWS.config.region).to be_eql('ap-northeast-1')
+      expect(Aws.config[:region]).to be_eql('ap-northeast-1')
+    end
+  end
+
+  describe '#ec2' do
+
+    it 'can return a client with regional endpoint' do
+      launcher = Stemcell::Launcher.new({'region' => 'us-east-1', 'ec2_endpoint' => nil})
+      client = launcher.send(:ec2)
+      expect(client.config[:endpoint].to_s).to be_eql('https://ec2.us-east-1.amazonaws.com')
+    end
+
+    it 'can return a client with custom endpoint' do
+      launcher = Stemcell::Launcher.new({
+        'region' => 'region1',
+        'ec2_endpoint' => 'https://endpoint1',
+      })
+      client = launcher.send(:ec2)
+      expect(client.config[:endpoint].to_s).to be_eql('https://endpoint1')
     end
   end
 end

data/spec/lib/stemcell/metadata_source/configuration_spec.rb

@@ -32,7 +32,9 @@ describe Stemcell::MetadataSource::Configuration do
       it "sets backing_store_options" do
         expect(config.backing_store_options).to eql({
           'instance_store' => {
-            'image_id' => 'ami-d9d6a6b0'
+            'us-east-1' => {
+              'image_id' => 'ami-d9d6a6b0'
+            }
           }
         })
       end
@@ -86,10 +88,11 @@ describe Stemcell::MetadataSource::Configuration do
 
   describe '#options_for_backing_store' do
     let(:backing_store) { 'instance_store' }
+    let(:region) { 'us-east-1' }
 
     context "when the backing store definition exists" do
       it "returns the options" do
-        expect(config.options_for_backing_store(backing_store)).to eql({
+        expect(config.options_for_backing_store(backing_store, region)).to eql({
           'image_id' => 'ami-d9d6a6b0'
         })
       end
@@ -97,13 +100,23 @@ describe Stemcell::MetadataSource::Configuration do
 
     context "when the backing store isn't defined" do
       let(:backing_store) { 'nyanstore' }
+      let(:region) { 'us-east-1' }
       it "raises" do
-        expect { config.options_for_backing_store(backing_store) }.to raise_error(
+        expect { config.options_for_backing_store(backing_store, region) }.to raise_error(
           Stemcell::UnknownBackingStoreError
         )
       end
     end
 
+    context "when the legacy backing store definition exists" do
+      let(:config_filename) { 'stemcell-backing-store-legacy.json' }
+      it "returns the options" do
+        expect(config.options_for_backing_store(backing_store, region)).to eql({
+          'image_id' => 'ami-d9d6a6b0'
+        })
+      end
+    end
+
   end
 
   describe '#random_az_in_region' do

data/spec/lib/stemcell/metadata_source_spec.rb

@@ -161,7 +161,8 @@ describe Stemcell::MetadataSource do
         it "calls the config object to retrieve the backing store options" do
           backing_options.merge!('image_id' => 'ami-nyancat')
           override_options.merge!('backing_store' => 'ebs')
-          expect(config).to receive(:options_for_backing_store).with('ebs') { backing_options }
+          override_options.merge!('region' => 'us-east-1')
+          expect(config).to receive(:options_for_backing_store).with('ebs', 'us-east-1') { backing_options }
           expect(expansion['image_id']).to eql 'ami-nyancat'
         end
 

data/stemcell.gemspec

@@ -17,8 +17,11 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'aws-sdk-v1', '~> 1.63'
-  s.add_runtime_dependency 'net-ssh',    '~> 2.9'
+  # pins several aws sdk transitive dependencies to maintain compatibility with Ruby < 2.3
+  s.add_runtime_dependency 'aws-eventstream', '~> 1.1.1'
+  s.add_runtime_dependency 'aws-sdk-ec2',     '~> 1'
+  s.add_runtime_dependency 'aws-sigv4',       '~> 1.2.4'
+  s.add_runtime_dependency 'net-ssh',         '~> 2.9'
   if RUBY_VERSION >= '2.0'
     s.add_runtime_dependency 'chef',     '>= 11.4.0'
   else
@@ -29,7 +32,7 @@ Gem::Specification.new do |s|
   # version dependency. lets explicitly include it here. if this becomes
   # no-longer a dependency of chef via chef-zero, then remove it
   s.add_runtime_dependency 'rack', '< 2.0.0'
-  s.add_runtime_dependency 'nokogiri', '< 1.7.0'
+  s.add_runtime_dependency 'nokogiri', '~> 1.8.2'
   s.add_runtime_dependency 'ffi-yajl', '< 2.3.1'
 
   s.add_runtime_dependency 'trollop',    '~> 2.1'