bosh-stemcell 1.5.0.pre.1113

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--profile

data/README.md

@@ -0,0 +1,104 @@
+# bosh-stemcell
+
+Tools for creating stemcells
+
+## Building a stemcell
+
+#### Once-off manual steps:
+
+0. Upload a keypair called "bosh" to AWS that you'll use to connect to the remote vm later
+0. Create "bosh-stemcell" security group on AWS to allow SSH access to the stemcell (once per AWS account)
+0. Add instructions to set BOSH_AWS_... environment variables
+0. Install the vagrant plugins we use:
+
+        vagrant plugin install vagrant-aws       --plugin-version 0.3.0
+
+#### Bring up the vagrant stemcell building VM
+
+From a fresh copy of the bosh repo:
+
+    export BOSH_AWS_ACCESS_KEY_ID=YOUR-AWS-ACCESS-KEY
+    export BOSH_AWS_SECRET_ACCESS_KEY=YOUR-AWS-SECRET-KEY
+    cd bosh-stemcell
+    vagrant up local
+
+#### Build the stemcell from inside the VM
+
+    vagrant ssh -c '
+      cd /bosh
+      bundle install --local
+      CANDIDATE_BUILD_NUMBER=1045 http_proxy=http://localhost:3142/ bundle exec rake ci:build_stemcell[vsphere,centos]
+    ' local
+
+# Run the stemcell locally with Fusion
+
+VMware Fusion is the preferred local virtual environment.  You can [purchase it here](http://www.vmware.com/products/fusion/).  Once you have Fusion installed:
+
+* Unpack the stemcell tarball.
+* Rename the new `image` file to `image.tgz` and untar that as well.  You should now have an `image.ovf` file.
+* Start VMware Fusion and import this file (**File** -> **Import**)
+* Save the imported OVF as `image.vmwarevm` in the `tmp` folder.
+* You should see a new VM in your VM library.
+* Double click that new VM and it should be booted.
+
+## Add the NIC
+
+Before starting the VM in VirtualBox:
+
+1. Add a network interface (**Virtual Machine** > **Settings** > **Add Device...** > **Network Adapter**)
+1. Select NAT networking (the default)
+
+# Run the stemcell locally with VirtualBox
+
+## Import the VM
+
+- The stemcell is dropped into `bosh/tmp/bosh-stemcell-???-vsphere-esxi-centos.tgz`.
+- Untar that file.  You now have a file named `image`.
+- Rename that to `image.tgz`, and untar *that* file.  You now have a file named `image.ovf`.
+- Double click on that file to open it in VirtualBox.
+
+## Add the NIC
+
+Before starting the VM in VirtualBox:
+
+1. Add a network interface (**Settings** > **Network** > **Adapter 1** > **Enable**)
+1. Select NAT networking (the default)
+1. Click on advanced
+1. Click on Port Forwarding and enable the following rule:
+    * Name: SSH
+    * Protocol: TCP
+    * Host IP: 127.0.0.1
+    * Host Port: 3333
+    * Guest IP: blank
+    * Guest Port: 22
+
+# Boot the VM
+
+Save the configuration and boot the VM.  Once you're booted, login as `root`/`c1oudc0w`.
+
+## Configure network locally
+
+### Fusion
+
+```bash
+$ grep subnet /Library/Preferences/VMware\ Fusion/*/dhcpd.conf
+/Library/Preferences/VMware Fusion/vmnet1/dhcpd.conf:subnet 172.16.0.0 netmask 255.255.255.0 {
+/Library/Preferences/VMware Fusion/vmnet8/dhcpd.conf:subnet 172.16.210.0 netmask 255.255.255.0 {
+```
+
+```bash
+ifconfig eth0 172.16.210.30/24 up
+route add default gw 172.16.210.2 eth0
+```
+
+### VirtualBox
+
+You'll want to pick an IP that's not in use by your stemcell building vm. 10.0.2.30 *should* be fine.
+
+```bash
+ifconfig eth0 10.0.2.30/24 up
+route add default gw 10.0.2.2 eth0
+```
+
+Test the network with `ping 8.8.8.8`
+

data/Vagrantfile

@@ -0,0 +1,37 @@
+Vagrant.configure('2') do |config|
+  env = ENV.to_hash
+  config.vm.box = 'bosh-stemcell'
+  config.ssh.username = 'ubuntu'
+
+  config.vm.define :local do |local|
+    local.vm.box_url = 'https://bosh-jenkins-artifacts.s3.amazonaws.com/bosh-vagrant-boxes/bosh-stemcell-virtualbox.box'
+    local.vm.provider :virtualbox do |virtualbox|
+      virtualbox.customize ['modifyvm', :id, '--cpus', '2']
+      virtualbox.customize ['modifyvm', :id, '--memory', '2048']
+    end
+  end
+
+  config.vm.define :remote do |remote|
+    remote.vm.box_url = 'https://bosh-jenkins-artifacts.s3.amazonaws.com/bosh-vagrant-boxes/bosh-stemcell-aws.box'
+    remote.vm.provider :aws do |aws, override|
+      aws.instance_type = 'c1.medium'
+      aws.access_key_id = env.fetch('BOSH_AWS_ACCESS_KEY_ID')
+      aws.secret_access_key = env.fetch('BOSH_AWS_SECRET_ACCESS_KEY')
+      aws.keypair_name = 'bosh'
+      aws.security_groups = ['bosh-stemcell']
+      aws.tags = { 'Name' => 'bosh-stemcell' }
+
+      override.ssh.username = 'ubuntu'
+      override.ssh.private_key_path = '~/.ssh/id_rsa_bosh'
+    end
+  end
+
+  config.vm.provision :shell do |shell|
+    shell.inline = <<-BASH
+    sudo mkdir -p /mnt/stemcells
+    sudo chown -R ubuntu /mnt/stemcells
+    BASH
+  end
+
+  config.vm.synced_folder '../', '/bosh'
+end

data/bosh-stemcell.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+version = File.read(File.expand_path('../../BOSH_VERSION', __FILE__)).strip
+
+Gem::Specification.new do |spec|
+  spec.name          = 'bosh-stemcell'
+  spec.version       = version
+  spec.authors       = 'Pivotal'
+  spec.email         = 'support@cloudfoundry.com'
+  spec.description   = 'Bosh::Stemcell provides tools to manage stemcells'
+  spec.summary       = 'Bosh::Stemcell provides tools to manage stemcells'
+  spec.homepage      = 'https://github.com/cloudfoundry/bosh'
+  spec.license       = 'Apache 2.0'
+
+  spec.required_ruby_version = Gem::Requirement.new('>= 1.9.3')
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = %w[lib]
+
+  spec.add_dependency 'bosh_aws_cpi', "~>#{version}"
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rspec-fire'
+  spec.add_development_dependency 'timecop'
+  spec.add_development_dependency 'serverspec'
+  spec.add_development_dependency 'foodcritic'
+end

data/lib/bosh/stemcell.rb

@@ -0,0 +1,4 @@
+module Bosh
+  module Stemcell
+  end
+end

data/lib/bosh/stemcell/archive.rb

@@ -0,0 +1,59 @@
+require 'rake/file_utils_ext'
+require 'yaml'
+require 'bosh/stemcell/aws/region'
+
+module Bosh::Stemcell
+  class Archive
+    attr_reader :path
+
+    def initialize(path = '')
+      @path = path
+      validate_stemcell
+    end
+
+    def manifest
+      @manifest ||= Psych.load(`tar -Oxzf #{path} stemcell.MF`)
+    end
+
+    def name
+      manifest.fetch('name')
+    end
+
+    def infrastructure
+      cloud_properties.fetch('infrastructure')
+    end
+
+    def version
+      cloud_properties.fetch('version')
+    end
+
+    def light?
+      infrastructure == 'aws' && ami_id
+    end
+
+    def ami_id(region = Aws::Region::DEFAULT)
+      cloud_properties.fetch('ami', {}).fetch(region, nil)
+    end
+
+    def extract(tar_options = {}, &block)
+      Dir.mktmpdir do |tmp_dir|
+        tar_cmd = "tar xzf #{path} --directory #{tmp_dir}"
+        tar_cmd << " --exclude=#{tar_options[:exclude]}" if tar_options.has_key?(:exclude)
+
+        Rake::FileUtilsExt.sh(tar_cmd)
+
+        block.call(tmp_dir, manifest)
+      end
+    end
+
+    private
+
+    def cloud_properties
+      manifest.fetch('cloud_properties')
+    end
+
+    def validate_stemcell
+      raise "Cannot find file `#{path}'" unless File.exists?(path)
+    end
+  end
+end

data/lib/bosh/stemcell/archive_filename.rb

@@ -0,0 +1,29 @@
+module Bosh::Stemcell
+  class ArchiveFilename
+    def initialize(version, infrastructure, operating_system, base_name, light)
+      @version = version
+      @infrastructure = infrastructure
+      @operating_system = operating_system
+      @base_name = base_name
+      @light = light
+    end
+
+    def to_s
+      stemcell_filename_parts = [name, version, infrastructure.name, infrastructure.hypervisor, operating_system.name]
+
+      "#{stemcell_filename_parts.join('-')}.tgz"
+    end
+
+    private
+
+    def name
+      light ? "light-#{base_name}" : base_name
+    end
+
+    attr_reader :base_name,
+                :version,
+                :infrastructure,
+                :operating_system,
+                :light
+  end
+end

data/lib/bosh/stemcell/aws/ami.rb

@@ -0,0 +1,58 @@
+require 'cloud'
+require 'bosh_aws_cpi'
+require 'ostruct'
+require 'yaml'
+require 'rake'
+require 'bosh/stemcell/aws/region'
+
+module Bosh::Stemcell::Aws
+  class Ami
+    attr_reader :stemcell
+
+    def initialize(stemcell, region)
+      @stemcell = stemcell
+      @region = region
+    end
+
+    def publish
+      cloud_config = OpenStruct.new(logger: Logger.new('ami.log'), task_checkpoint: nil)
+      Bosh::Clouds::Config.configure(cloud_config)
+
+      cloud = Bosh::Clouds::Provider.create('aws', options)
+
+      stemcell.extract do |tmp_dir, stemcell_manifest|
+        ami_id = cloud.create_stemcell("#{tmp_dir}/image", stemcell_manifest['cloud_properties'])
+        cloud.ec2.images[ami_id].public = true
+        ami_id
+      end
+    end
+
+    private
+
+    attr_reader :region
+
+    def options
+      # just fake the registry struct, as we don't use it
+      {
+        'aws' => aws,
+        'registry' => {
+          'endpoint' => 'http://fake.registry',
+          'user' => 'fake',
+          'password' => 'fake'
+        }
+      }
+    end
+
+    def aws
+      access_key_id = ENV['BOSH_AWS_ACCESS_KEY_ID']
+      secret_access_key = ENV['BOSH_AWS_SECRET_ACCESS_KEY']
+
+      {
+        'default_key_name' => 'fake',
+        'region' => region.name,
+        'access_key_id' => access_key_id,
+        'secret_access_key' => secret_access_key
+      }
+    end
+  end
+end

data/lib/bosh/stemcell/aws/light_stemcell.rb

@@ -0,0 +1,44 @@
+require 'rake/file_utils'
+require 'yaml'
+require 'common/deep_copy'
+require 'bosh/stemcell/aws/region'
+require 'bosh/stemcell/aws/ami'
+
+module Bosh::Stemcell::Aws
+  class LightStemcell
+    def initialize(stemcell)
+      @stemcell = stemcell
+    end
+
+    def write_archive
+      stemcell.extract(exclude: 'image') do |extracted_stemcell_dir|
+        Dir.chdir(extracted_stemcell_dir) do
+          FileUtils.touch('image', verbose: true)
+
+          File.open('stemcell.MF', 'w') do |out|
+            Psych.dump(manifest, out)
+          end
+
+          Rake::FileUtilsExt.sh("sudo tar cvzf #{path} *")
+        end
+      end
+    end
+
+    def path
+      File.join(File.dirname(stemcell.path), "light-#{File.basename(stemcell.path)}")
+    end
+
+    private
+
+    attr_reader :stemcell
+
+    def manifest
+      region = Region.new
+      ami = Ami.new(stemcell, region)
+      ami_id = ami.publish
+      manifest = Bosh::Common::DeepCopy.copy(stemcell.manifest)
+      manifest['cloud_properties']['ami'] = { region.name => ami_id }
+      manifest
+    end
+  end
+end

data/lib/bosh/stemcell/aws/region.rb

@@ -0,0 +1,15 @@
+require 'net/http'
+
+module Bosh
+  module Stemcell
+    module Aws
+      class Region
+        DEFAULT = 'us-east-1'
+
+        def name
+          Net::HTTP.get('169.254.169.254', '/latest/meta-data/placement/availability-zone').chop
+        end
+      end
+    end
+  end
+end

data/lib/bosh/stemcell/builder_command.rb

@@ -0,0 +1,174 @@
+require 'fileutils'
+
+require 'bosh/core/shell'
+require 'bosh/stemcell/builder_options'
+require 'bosh/stemcell/disk_image'
+require 'bosh/stemcell/infrastructure'
+require 'bosh/stemcell/operating_system'
+require 'bosh/stemcell/stage_collection'
+require 'bosh/stemcell/stage_runner'
+
+module Bosh::Stemcell
+  class BuilderCommand
+    def initialize(env, options)
+      @environment = env
+      @infrastructure = Infrastructure.for(options.fetch(:infrastructure_name))
+      @operating_system = OperatingSystem.for(options.fetch(:operating_system_name))
+
+      @stemcell_builder_options = BuilderOptions.new(
+        env,
+        tarball: options.fetch(:release_tarball_path),
+        stemcell_version: options.fetch(:version),
+        infrastructure: infrastructure,
+        operating_system: operating_system,
+      )
+      @shell = Bosh::Core::Shell.new
+    end
+
+    def build
+      sanitize
+
+      prepare_build_root
+
+      prepare_build_path
+
+      copy_stemcell_builder_to_build_path
+
+      prepare_work_root
+
+      persist_settings_for_bash
+
+      stage_collection = StageCollection.new(
+        infrastructure: infrastructure,
+        operating_system: operating_system
+      )
+      stage_runner = StageRunner.new(
+        build_path: build_path,
+        command_env: command_env,
+        settings_file: settings_path,
+        work_path: work_root
+      )
+      stage_runner.configure_and_apply(stage_collection.operating_system_stages)
+      stage_runner.configure_and_apply(stage_collection.infrastructure_stages)
+      system(rspec_command) || raise('Stemcell specs failed')
+
+      stemcell_file
+    end
+
+    def chroot_dir
+      File.join(work_path, 'chroot')
+    end
+
+    private
+
+    attr_reader(
+      :shell,
+      :environment,
+      :infrastructure,
+      :operating_system,
+      :stemcell_builder_options
+    )
+
+    def rspec_command
+      [
+        "cd #{File.expand_path('../../..', File.dirname(__FILE__))};",
+        "STEMCELL_IMAGE=#{image_file_path}",
+        'bundle exec rspec -fd',
+        "spec/stemcells/#{operating_system.name}_spec.rb",
+        "spec/stemcells/#{infrastructure.name}_spec.rb",
+      ].join(' ')
+    end
+
+    def image_file_path
+      File.join(work_path, settings['stemcell_image_name'])
+    end
+
+    def image_mount_point
+      File.join(work_path, 'mnt')
+    end
+
+    def sanitize
+      FileUtils.rm_rf('*.tgz')
+
+      system("sudo umount #{File.join(work_path, 'mnt/tmp/grub', settings['stemcell_image_name'])} 2> /dev/null")
+      system("sudo umount #{image_mount_point} 2> /dev/null")
+      system("sudo rm -rf #{base_directory}")
+    end
+
+    def settings
+      stemcell_builder_options.default
+    end
+
+    def base_directory
+      File.join('/mnt', 'stemcells', infrastructure.name, infrastructure.hypervisor, operating_system.name)
+    end
+
+    def build_root
+      File.join(base_directory, 'build')
+    end
+
+    def work_root
+      File.join(base_directory, 'work')
+    end
+
+    def prepare_build_root
+      FileUtils.mkdir_p(build_root, verbose: true)
+    end
+
+    def prepare_work_root
+      FileUtils.mkdir_p(work_root, verbose: true)
+    end
+
+    def build_path
+      File.join(build_root, 'build')
+    end
+
+    def work_path
+      File.join(work_root, 'work')
+    end
+
+    def prepare_build_path
+      FileUtils.rm_rf(build_path, verbose: true) if Dir.exists?(build_path)
+      FileUtils.mkdir_p(build_path, verbose: true)
+    end
+
+    def stemcell_builder_source_dir
+      File.join(File.expand_path('../../../../..', __FILE__), 'stemcell_builder')
+    end
+
+    def copy_stemcell_builder_to_build_path
+      FileUtils.cp_r(Dir.glob("#{stemcell_builder_source_dir}/*"), build_path, preserve: true, verbose: true)
+    end
+
+    def settings_path
+      File.join(build_path, 'etc', 'settings.bash')
+    end
+
+    def persist_settings_for_bash
+      File.open(settings_path, 'a') do |f|
+        f.printf("\n# %s\n\n", '=' * 20)
+        settings.each do |k, v|
+          f.print "#{k}=#{v}\n"
+        end
+      end
+    end
+
+    def command_env
+      "env #{hash_as_bash_env(proxy_settings_from_environment)}"
+    end
+
+    def stemcell_file
+      File.join(work_path, settings['stemcell_tgz'])
+    end
+
+    def proxy_settings_from_environment
+      keep = %w(HTTP_PROXY NO_PROXY)
+
+      environment.select { |k| keep.include?(k.upcase) }
+    end
+
+    def hash_as_bash_env(env)
+      env.map { |k, v| "#{k}='#{v}'" }.join(' ')
+    end
+  end
+end