bosh-stemcell 1.2922.0 → 1.2941.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e63abd8e3e4094e14eb2055f037830ebf0fdfb8b
-  data.tar.gz: 4adef86bd8f5af990a1bd68f19a487e22f89c99a
+  metadata.gz: 23245ec8c50d7c96d9a78f715e36370ededb3c3c
+  data.tar.gz: 536b9148c3ba2bd1c2b4da7d74f942a80c15e4ac
 SHA512:
-  metadata.gz: 883740b6c50493eb018fad4ec6cf2c3012c46aff7d5e4c0aad8f4e7dd9ca645a8715bac54c254e0eae4f654cd7e214abd0e1de59951f720feec68ca194a25087
-  data.tar.gz: 317890a2a52246297e56c4b87d1b5cd7800e7dfd58d652d295727be725822acf4ba7b0283187635119c9ce51b2655dd2553cd46bf390f3c1d42c34398aa3b657
+  metadata.gz: 91d2d26a75df2d1b5d3778438e58c496e1dc45be4e11f779896c9ee3cacf714ce7968382d34ca7f34430ea4f2644504b167e4c9dd9563d92efe2ccfd510f6301
+  data.tar.gz: e38fd4b4de070bbbdd837d9212f6efaccaf363f2b555ba8fe4ffa3a0f1d3e309df97475c9ff29a8af5401d00e7abf83e3e634dbd9e759d1d25bf02c5295d3887

data/lib/bosh/stemcell/archive.rb

@@ -34,11 +34,7 @@ module Bosh::Stemcell
     end
 
     def light?
-      infrastructure == 'aws' && ami_id
-    end
-
-    def ami_id(region = Aws::Region::DEFAULT)
-      cloud_properties.fetch('ami', {}).fetch(region, nil)
+      infrastructure == 'aws' && has_ami?
     end
 
     def extract(tar_options = {}, &block)
@@ -54,6 +50,10 @@ module Bosh::Stemcell
 
     private
 
+    def has_ami?
+      cloud_properties.has_key? 'ami'
+    end
+
     def cloud_properties
       manifest.fetch('cloud_properties')
     end

data/lib/bosh/stemcell/aws/ami_collection.rb

@@ -0,0 +1,143 @@
+require 'cloud'
+require 'bosh_aws_cpi'
+require 'ostruct'
+require 'yaml'
+require 'rake'
+
+module Bosh::Stemcell::Aws
+  class AmiCollection
+    MAX_COPY_IMAGE_WAIT_ATTEMPTS = 360
+
+    attr_reader :stemcell
+
+    def initialize(stemcell, regions, virtualization_type)
+      @stemcell = stemcell
+      @seed_region = regions.first
+      @dest_regions = regions - [@seed_region]
+      @virtualization_type = virtualization_type
+
+      @access_key_id = ENV['BOSH_AWS_ACCESS_KEY_ID']
+      @secret_access_key = ENV['BOSH_AWS_SECRET_ACCESS_KEY']
+    end
+
+    def publish
+      logger = Logger.new('ami.log')
+      cloud_config = OpenStruct.new(logger: logger, task_checkpoint: nil)
+      Bosh::Clouds::Config.configure(cloud_config)
+      cloud = Bosh::Clouds::Provider.create(cloud_options, 'fake-director-uuid')
+
+      region_ami_mapping = {}
+      @stemcell.extract do |tmp_dir, stemcell_manifest|
+        cloud_properties = stemcell_manifest['cloud_properties'].merge(
+          'virtualization_type' => @virtualization_type
+        )
+
+        seed_ami_id = cloud.create_stemcell("#{tmp_dir}/image", cloud_properties)
+        seed_ami = cloud.ec2.images[seed_ami_id]
+        seed_ami.public = true
+        region_ami_mapping = copy_to_regions(logger, seed_ami_id, seed_ami.name, seed_ami.tags)
+        region_ami_mapping[@seed_region] = seed_ami_id
+      end
+
+      region_ami_mapping
+    end
+
+    private
+
+    def copy_to_regions(logger, source_id, source_name, source_tags)
+      threads = []
+      mutex = Mutex.new
+      region_ami_mapping = {}
+
+      @dest_regions.each do |dest_region|
+        threads << Thread.new do
+          copied_ami_id = copy_to_region(logger, source_id, source_name, source_tags, @seed_region, dest_region)
+          mutex.synchronize { region_ami_mapping[dest_region] = copied_ami_id }
+        end
+      end
+
+      threads.each { |t| t.join }
+      region_ami_mapping
+    end
+
+    def copy_to_region(logger, source_ami_id, source_ami_name, source_ami_tags, source_region, dest_region)
+      logger.info "Copying AMI '#{source_ami_id}' from region '#{source_region}' to region '#{dest_region}'"
+
+      client_options = {
+        :access_key_id => @access_key_id,
+        :secret_access_key => @secret_access_key,
+        :region => dest_region
+      }
+      query_client = AWS::EC2::Client.new(client_options)
+      ec2_client = AWS::EC2.new(client_options)
+
+      copied_ami_id = copy_image(ec2_client, query_client, source_region, source_ami_id, source_ami_name)
+      set_image_attributes(ec2_client, copied_ami_id, source_ami_tags)
+      logger.info "Finished copying AMI '#{source_ami_id}' from region '#{source_region}'" +
+          " to AMI '#{copied_ami_id}' in region '#{dest_region}'"
+
+      copied_ami_id
+    end
+
+    def copy_image(ec2_client, query_client, source_region, source_ami_id, source_ami_name)
+      copy_image_options = {
+        source_region: source_region,
+        source_image_id: source_ami_id,
+        name: source_ami_name
+      }
+
+      copied_ami_id = query_client.copy_image(copy_image_options)[:image_id]
+      # we have to wait for the image to be available in order to set attributes on it
+      wait_for_ami_to_be_available(ec2_client, copied_ami_id)
+      copied_ami_id
+    end
+
+    def set_image_attributes(ec2_client, ami_id, ami_tags)
+      ami = ec2_client.images[ami_id]
+
+      ami.public = true
+      ami.add_tag('Name', :value => ami_tags['Name'])
+    end
+
+    def wait_for_ami_to_be_available(ec2, ami_id)
+      # AMI is likely to be in state :pending or it may not be found (NotFound error).
+      image_state = lambda {
+        begin
+          ec2.images[ami_id].state
+        rescue AWS::EC2::Errors::InvalidAMIID::NotFound
+          :not_found
+        end
+      }
+
+      attempts = 0
+      until image_state.call == :available
+        if attempts > MAX_COPY_IMAGE_WAIT_ATTEMPTS
+          raise "Timed out waiting for AMI '#{ami_id}' to reach 'available' state"
+        end
+        attempts += 1
+        sleep(0.5*attempts)
+      end
+    end
+
+    # At some point we should extract the logic in cloud.create_stemcell into a library which can be used here
+    # it doesn't make a lot of sense to new up a set of options of the registry.
+    def cloud_options
+      {
+        'plugin' => 'aws',
+        'properties' => {
+          'aws' => {
+            'access_key_id' => @access_key_id,
+            'secret_access_key' => @secret_access_key,
+            'region' => @seed_region,
+            'default_key_name' => 'fake'
+          },
+          'registry' => {
+            'endpoint' => 'http://fake.registry',
+            'user' => 'fake',
+            'password' => 'fake'
+          }
+        }
+      }
+    end
+  end
+end

data/lib/bosh/stemcell/aws/light_stemcell.rb

@@ -1,53 +1,52 @@
 require 'rake/file_utils'
 require 'yaml'
 require 'common/deep_copy'
+require 'bosh/stemcell/aws/ami_collection'
 require 'bosh/stemcell/aws/region'
-require 'bosh/stemcell/aws/ami'
 
 module Bosh::Stemcell::Aws
   HVM_VIRTUALIZATION = 'hvm'
 
   class LightStemcell
-    def initialize(stemcell, virtualization_type)
+    def initialize(stemcell, virtualization_type, regions=Region::REGIONS)
       @stemcell = stemcell
       @virtualization_type = virtualization_type
+      @regions = regions
     end
 
     def write_archive
-      stemcell.extract(exclude: 'image') do |extracted_stemcell_dir|
+      @stemcell.extract(exclude: 'image') do |extracted_stemcell_dir|
         Dir.chdir(extracted_stemcell_dir) do
           FileUtils.touch('image', verbose: true)
-
-          File.open('stemcell.MF', 'w') do |out|
-            Psych.dump(manifest, out)
-          end
-
+          File.write('stemcell.MF', Psych.dump(manifest))
           Rake::FileUtilsExt.sh("sudo tar cvzf #{path} *")
         end
       end
     end
 
     def path
-      stemcell_name = File.basename(stemcell.path)
-      stemcell_name = stemcell_name.gsub("xen", "xen-hvm") if virtualization_type == HVM_VIRTUALIZATION
-      File.join(File.dirname(stemcell.path), "light-#{stemcell_name}")
+      stemcell_name = adjust_hvm_name(File.basename(@stemcell.path))
+      File.join(File.dirname(@stemcell.path), "light-#{stemcell_name}")
     end
 
     private
 
-    attr_reader :stemcell, :virtualization_type
-
+    # this method has heavy side effects
     def manifest
-      region = Region.new
-      ami = Ami.new(stemcell, region, virtualization_type)
-      ami_id = ami.publish
-      manifest = Bosh::Common::DeepCopy.copy(stemcell.manifest)
-      if virtualization_type == HVM_VIRTUALIZATION
-        manifest['name'] = manifest['name'].gsub("xen", "xen-hvm")
-        manifest['cloud_properties']['name'] = manifest['cloud_properties']['name'].gsub("xen", "xen-hvm")
-      end
-      manifest['cloud_properties']['ami'] = { region.name => ami_id }
+      manifest = Bosh::Common::DeepCopy.copy(@stemcell.manifest)
+      manifest['name'] = adjust_hvm_name(manifest['name'])
+      manifest['cloud_properties']['name'] = adjust_hvm_name(manifest['cloud_properties']['name'])
+
+      ami_collection = AmiCollection.new(@stemcell, @regions, @virtualization_type)
+
+      # Light stemcell contains AMIs for all regions
+      # so that CPI can pick one based on its configuration
+      manifest['cloud_properties']['ami'] = ami_collection.publish
       manifest
     end
+
+    def adjust_hvm_name(name)
+      @virtualization_type == HVM_VIRTUALIZATION ? name.gsub("xen", "xen-hvm") : name
+    end
   end
 end

data/lib/bosh/stemcell/aws/region.rb

@@ -5,10 +5,10 @@ module Bosh
     module Aws
       class Region
         DEFAULT = 'us-east-1'
-
-        def name
-          Net::HTTP.get('169.254.169.254', '/latest/meta-data/placement/availability-zone').chop
-        end
+        REGIONS = %w{
+          us-east-1 us-west-1 us-west-2 eu-west-1 eu-central-1
+          ap-southeast-1 ap-southeast-2 ap-northeast-1 sa-east-1
+        }
       end
     end
   end

data/lib/bosh/stemcell/stage_collection.rb

@@ -201,6 +201,7 @@ module Bosh::Stemcell
           :rsyslog_config,
           :delay_monit_start,
           :system_grub,
+          :cron_config,
       ].flatten
 
       if operating_system.version.to_f < 7
@@ -220,6 +221,7 @@ module Bosh::Stemcell
         :delay_monit_start,
         :system_grub,
         :rhel_unsubscribe,
+        :cron_config,
       ].flatten
     end
 
@@ -240,6 +242,7 @@ module Bosh::Stemcell
         :delay_monit_start,
         :system_grub,
         :vim_tiny,
+        :cron_config,
       ].flatten
     end
 

data/lib/bosh/stemcell/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Stemcell
-    VERSION = '1.2922.0'
+    VERSION = '1.2941.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-stemcell
 version: !ruby/object:Gem::Version
-  version: 1.2922.0
+  version: 1.2941.0
 platform: ruby
 authors:
 - Pivotal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-10 00:00:00.000000000 Z
+date: 2015-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2922.0
+        version: 1.2941.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2922.0
+        version: 1.2941.0
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2922.0
+        version: 1.2941.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2922.0
+        version: 1.2941.0
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
@@ -176,7 +176,7 @@ files:
 - lib/bosh/stemcell/archive.rb
 - lib/bosh/stemcell/archive_filename.rb
 - lib/bosh/stemcell/archive_handler.rb
-- lib/bosh/stemcell/aws/ami.rb
+- lib/bosh/stemcell/aws/ami_collection.rb
 - lib/bosh/stemcell/aws/light_stemcell.rb
 - lib/bosh/stemcell/aws/region.rb
 - lib/bosh/stemcell/build_environment.rb

data/lib/bosh/stemcell/aws/ami.rb

@@ -1,65 +0,0 @@
-require 'cloud'
-require 'bosh_aws_cpi'
-require 'ostruct'
-require 'yaml'
-require 'rake'
-require 'bosh/stemcell/aws/region'
-
-module Bosh::Stemcell::Aws
-  class Ami
-    attr_reader :stemcell
-
-    def initialize(stemcell, region, virtualization_type)
-      @stemcell = stemcell
-      @region = region
-      @virtualization_type = virtualization_type || 'paravirtual'
-    end
-
-    def publish
-      cloud_config = OpenStruct.new(logger: Logger.new('ami.log'), task_checkpoint: nil)
-      Bosh::Clouds::Config.configure(cloud_config)
-
-      cloud = Bosh::Clouds::Provider.create(options, 'fake-director-uuid')
-
-      stemcell.extract do |tmp_dir, stemcell_manifest|
-        cloud_properties = stemcell_manifest['cloud_properties'].merge(
-          'virtualization_type' => virtualization_type
-        )
-        ami_id = cloud.create_stemcell("#{tmp_dir}/image", cloud_properties)
-        cloud.ec2.images[ami_id].public = true
-        ami_id
-      end
-    end
-
-    private
-
-    attr_reader :region, :virtualization_type
-
-    def options
-      # just fake the registry struct, as we don't use it
-      {
-        'plugin' => 'aws',
-        'properties' => {
-          'aws' => aws,
-          'registry' => {
-            'endpoint' => 'http://fake.registry',
-            'user' => 'fake',
-            'password' => 'fake'
-          }
-        }
-      }
-    end
-
-    def aws
-      access_key_id = ENV['BOSH_AWS_ACCESS_KEY_ID']
-      secret_access_key = ENV['BOSH_AWS_SECRET_ACCESS_KEY']
-
-      {
-        'default_key_name' => 'fake',
-        'region' => region.name,
-        'access_key_id' => access_key_id,
-        'secret_access_key' => secret_access_key
-      }
-    end
-  end
-end