bosh-bootstrap 0.13.2 → 0.14.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MTk4NzJmZDdmNjEzMmEzZjQ5YmE1ZjFjNTc4NjZkZDVkOGZkYWQyZQ==
-  data.tar.gz: !binary |-
-    NzljOTRmMTU5MzgzMGRlMmE3YWE2MTdiMGFjYWYwNzliYzk1YzU4NQ==
+SHA1:
+  metadata.gz: b5ea6f814d89322a4ce8eecc1e9e56d6a8c77578
+  data.tar.gz: a9808b7ddf7eed80eca2fdb9cdf5f973bd14916e
 SHA512:
-  metadata.gz: !binary |-
-    OGMzN2FkOTc5YjFjNTY0NDMwZWIxZmU3NTU1MjM2NWEzOWU2MWYwZGYxMmNi
-    ZjY4NWMzZmM5MGQzM2NlMjAzODQxZjBkNDBmMTIxOGI3NzY4YTNiMjEzNWM3
-    Mzk3MWIxZDVhZmNjODZjYTMzM2Q4ZDQzNWU0NmQ0ZjIyZWRmYzY=
-  data.tar.gz: !binary |-
-    MDQ0NmYzMDU4MmM3Mzg5YzFhNTFkMzlmNzI4MmY5M2JiMGMwMjhmMWY1ODc4
-    NjhjMjJiNWZmMDViYzM0MGU5YmM3Y2Y5YTg2ZjZhNTFjMThmMGI0ZWYxNzg4
-    ZDExYzIyYTA0ODY2ZDkxYzZiZjYxNjAxNTlkMDNjY2UyZDY3YzA=
+  metadata.gz: 91325833e973b0ca5dbf52d9d5b7090bd7bc2c982169e0c15eab750b3da66b968898ab303cfbfc9a5c03c294c0303786a2aa66c0f8c7466444a0c21c508af53f
+  data.tar.gz: a3e81665b3fcaf6617520af632c2311a5dbcaec9eb27ed3de125964dcdaa33c5c61bfeb683338d31f3f39fbad3a6b33608136796dc57e0b0123eb4432693f88b

data/ChangeLog.md

@@ -8,6 +8,12 @@ gem install bosh-bootstrap
 bosh-bootstrap deploy
 ```
 
+v0.14
+
+- default target directory is current director, rather than `~/.microbosh`
+- [aws] VPC support - detects if VPCs available and allows selection, then select of subset, then creates security groups into the VPC
+- [aws] Using m3.medium instead of m1.medium
+
 v0.13
 -----
 

data/bosh-bootstrap.gemspec

@@ -22,10 +22,12 @@ EOS
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "cyoi", "~> 0.10"
+  gem.add_dependency "cyoi", "~> 0.11"
   gem.add_dependency "fog", "~> 1.11"
   gem.add_dependency "readwritesettings", "~> 3.0"
   gem.add_dependency "thor", "~> 0.18"
+  gem.add_dependency "httpclient", '=2.4.0'
+
   gem.add_dependency "redcard"
   gem.add_dependency "rbvmomi"
   gem.add_development_dependency "rake"

data/lib/bosh-bootstrap/cli/commands/deploy.rb

@@ -42,7 +42,7 @@ class Bosh::Bootstrap::Cli::Commands::Deploy
     reload_settings!
 
     # TODO why passing provider_client rather than a Cyoi::Cli::Network object?
-    network = Bosh::Bootstrap::Network.new(settings.provider.name, cyoi_provider_client)
+    network = Bosh::Bootstrap::Network.new(settings.provider.name, cyoi_provider_client, settings)
     network.deploy
   end
 
@@ -59,14 +59,14 @@ class Bosh::Bootstrap::Cli::Commands::Deploy
   # download if stemcell
   def select_public_image_or_download_stemcell
     print "Determining stemcell image/file to use... "
-    stemcell = microbosh_provider.stemcell
-    while (stemcell || "").size == 0
+    stemcell_path = microbosh_provider.stemcell_path
+    while (stemcell_path || "").size == 0
       puts "failed. Retrying..."
       print "Determining stemcell image/file to use... "
-      stemcell = microbosh_provider.stemcell
+      stemcell_path = microbosh_provider.stemcell_path
     end
-    settings.set("bosh.stemcell", stemcell)
-    puts settings.bosh.stemcell
+    settings.set("bosh.stemcell_path", stemcell_path)
+    puts settings.bosh.stemcell_path
   end
 
   def perform_microbosh_deploy

data/lib/bosh-bootstrap/cli/helpers/settings.rb

@@ -7,10 +7,10 @@ module Bosh::Bootstrap::Cli::Helpers::Settings
   # The base directory for holding the manifest settings file
   # and private keys
   #
-  # Defaults to ~/.bosh_inception; and can be overridden with either:
+  # Defaults to current directory; and can be overridden:
   # * $SETTINGS - to a folder (supported method)
   def settings_dir
-    @settings_dir ||= File.expand_path(ENV["SETTINGS"] || "~/.microbosh")
+    @settings_dir ||= File.expand_path(ENV["SETTINGS"] || ".")
   end
 
   def settings_dir=(settings_dir)

data/lib/bosh-bootstrap/microbosh.rb

@@ -40,7 +40,7 @@ class Bosh::Bootstrap::Microbosh
     chdir(base_path) do
       setup_gems
       create_microbosh_yml(settings)
-      deploy_or_update(settings.bosh.name, settings.bosh.stemcell)
+      deploy_or_update(settings.bosh.name, settings.bosh.stemcell_path)
     end
   end
 
@@ -52,7 +52,6 @@ class Bosh::Bootstrap::Microbosh
       f << <<-RUBY
 source 'https://rubygems.org'
 
-gem "bosh-bootstrap", path: "#{gempath}"
 gem "bosh_cli_plugin_micro"
       RUBY
     end
@@ -70,4 +69,4 @@ gem "bosh_cli_plugin_micro"
       bundle "exec bosh -n micro deploy --update-if-exists", stemcell
     end
   end
-end
+end

data/lib/bosh-bootstrap/microbosh_providers/aws.rb

@@ -2,18 +2,6 @@ require "bosh-bootstrap/microbosh_providers/base"
 
 module Bosh::Bootstrap::MicroboshProviders
   class AWS < Base
-    # if us-east-1 -> ami
-    # if not running in target aws region -> error "Must either use us-east-1 or run 'bosh bootstrap deploy' within target AWS region"
-    # else download stemcell & return path
-    def stemcell
-      unless settings.exists?("bosh.stemcell")
-        if ami_region?
-          fetch_ami
-        else
-          download_stemcell
-        end
-      end
-    end
 
     def to_hash
       data = super.merge({
@@ -41,7 +29,7 @@ module Bosh::Bootstrap::MicroboshProviders
     end
 
     def resources_cloud_properties
-      {"instance_type"=>"m1.medium"}
+      {"instance_type"=>"m3.medium"}
     end
 
     def cloud_properties
@@ -64,22 +52,28 @@ module Bosh::Bootstrap::MicroboshProviders
       settings.provider.region
     end
 
-    # only us-east-1 has AMIs published currently
-    def ami_region?
-      aws_region == "us-east-1"
-    end
-
-    def fetch_ami
-      Net::HTTP.get("#{jenkins_bucket}.s3.amazonaws.com", ami_uri_path(aws_region)).strip
+    # @return Bosh::Cli::PublicStemcell latest stemcell for aws/trusty
+    # If us-east-1 region, then return light stemcell
+    def latest_stemcell
+      @latest_stemcell ||= begin
+        trusty_stemcells = if light_stemcell?
+          recent_stemcells.select do |s|
+            s.name =~ /aws/ && s.name =~ /trusty/ && s.name =~ /^light/
+          end
+        else
+          recent_stemcells.select do |s|
+            s.name =~ /aws/ && s.name =~ /trusty/ && s.name =~ /^bosh/
+          end
+        end
+        trusty_stemcells.sort {|s1, s2| s2.version <=> s1.version}.first
+      end
     end
 
-    def ami_uri_path(region)
-      "/last_successful-bosh-stemcell-aws_ami_#{region}"
+    # only us-east-1 has light stemcells published
+    def light_stemcell?
+      aws_region == "us-east-1"
     end
 
-    def stemcell_uri
-      "http://bosh-jenkins-artifacts.s3.amazonaws.com/bosh-stemcell/aws/bosh-stemcell-latest-aws-xen-ubuntu.tgz"
-    end
   end
 end
 Bosh::Bootstrap::MicroboshProviders.register_provider("aws", Bosh::Bootstrap::MicroboshProviders::AWS)

data/lib/bosh-bootstrap/microbosh_providers/base.rb

@@ -1,4 +1,5 @@
 require "bosh-bootstrap/microbosh_providers"
+require "bosh-bootstrap/public_stemcells"
 
 # for the #sh helper
 require "rake"
@@ -56,13 +57,26 @@ class Bosh::Bootstrap::MicroboshProviders::Base
     "bosh-jenkins-artifacts"
   end
 
+  def stemcell_path
+    unless settings.exists?("bosh.stemcell_path")
+      download_stemcell
+    end
+  end
+
+  def recent_stemcells
+    @recent_stemcells ||= begin
+      public_stemcells = Bosh::Bootstrap::PublicStemcells.new
+      public_stemcells.recent
+    end
+  end
+
   # downloads latest stemcell & returns path
   def download_stemcell
     mkdir_p(stemcell_dir)
     chdir(stemcell_dir) do
-      stemcell_path = File.expand_path(File.basename(stemcell_uri))
+      stemcell_path = File.expand_path(latest_stemcell.name)
       unless File.exists?(stemcell_path)
-        sh "curl -O '#{stemcell_uri}'"
+        sh "curl -O '#{latest_stemcell.url}'"
       end
       return stemcell_path
     end

data/lib/bosh-bootstrap/microbosh_providers/openstack.rb

@@ -2,11 +2,6 @@ require "bosh-bootstrap/microbosh_providers/base"
 
 module Bosh::Bootstrap::MicroboshProviders
   class OpenStack < Base
-    def stemcell
-      unless settings.exists?("bosh.stemcell")
-        download_stemcell
-      end
-    end
 
     def to_hash
       super.merge({
@@ -126,9 +121,16 @@ module Bosh::Bootstrap::MicroboshProviders
       !!(settings.provider["options"] && settings.provider.options.boot_from_volume)
     end
 
-    def stemcell_uri
-      "http://bosh-jenkins-artifacts.s3.amazonaws.com/bosh-stemcell/openstack/bosh-stemcell-latest-openstack-kvm-ubuntu.tgz"
+    # @return Bosh::Cli::PublicStemcell latest stemcell for openstack/trusty
+    def latest_stemcell
+      @latest_stemcell ||= begin
+        trusty_stemcells = recent_stemcells.select do |s|
+          s.name =~ /openstack/ && s.name =~ /trusty/
+        end
+        trusty_stemcells.sort {|s1, s2| s2.version <=> s1.version}.first
+      end
     end
+
   end
 end
 Bosh::Bootstrap::MicroboshProviders.register_provider("openstack", Bosh::Bootstrap::MicroboshProviders::OpenStack)

data/lib/bosh-bootstrap/microbosh_providers/vsphere.rb

@@ -100,9 +100,16 @@ module Bosh::Bootstrap::MicroboshProviders
       }
     end
 
-    def stemcell_uri
-      "http://#{jenkins_bucket}.s3.amazonaws.com/bosh-stemcell/vsphere/bosh-stemcell-latest-vsphere-esxi-ubuntu.tgz"
+    # @return Bosh::Cli::PublicStemcell latest stemcell for vsphere/trusty
+    def latest_stemcell
+      @latest_stemcell ||= begin
+        trusty_stemcells = recent_stemcells.select do |s|
+          s.name =~ /vsphere/ && s.name =~ /trusty/
+        end
+        trusty_stemcells.sort {|s1, s2| s2.version <=> s1.version}.first
+      end
     end
+
   end
 end
 Bosh::Bootstrap::MicroboshProviders.register_provider("vsphere", Bosh::Bootstrap::MicroboshProviders::VSphere)

data/lib/bosh-bootstrap/network.rb

@@ -5,14 +5,16 @@ class Bosh::Bootstrap::Network
 
   attr_reader :provider_name
   attr_reader :cyoi_provider_client
+  attr_reader :settings
 
-  def initialize(provider_name, cyoi_provider_client)
+  def initialize(provider_name, cyoi_provider_client, settings)
     @provider_name = provider_name
     @cyoi_provider_client = cyoi_provider_client
+    @settings = settings
   end
 
   def deploy
-    network_provider.perform
+    network_provider.perform(settings)
   end
 
   # Attempt to load and instantiate a NetworkProviders class

data/lib/bosh-bootstrap/network_providers/aws.rb

@@ -6,9 +6,13 @@ module Bosh::Bootstrap::NetworkProviders
       @cyoi_provider_client = cyoi_provider_client
     end
 
-    def perform
+    def perform(settings)
+      attributes = {}
+      if vpc_id = settings.exists?("address.vpc_id")
+        attributes[:vpc_id] = vpc_id
+      end
       security_groups.each do |name, ports|
-        cyoi_provider_client.create_security_group(name.to_s, name.to_s, ports: ports)
+        cyoi_provider_client.create_security_group(name.to_s, name.to_s, {ports: ports}, attributes)
       end
     end
 

data/lib/bosh-bootstrap/network_providers/dummy.rb

@@ -3,7 +3,7 @@ module Bosh::Bootstrap::NetworkProviders
     def initialize(cyoi_provider_client)
     end
 
-    def perform
+    def perform(settings)
     end
   end
 end

data/lib/bosh-bootstrap/network_providers/openstack.rb

@@ -6,7 +6,7 @@ module Bosh::Bootstrap::NetworkProviders
       @cyoi_provider_client = cyoi_provider_client
     end
 
-    def perform
+    def perform(settings)
       cyoi_provider_client.create_security_group("ssh", "ssh", 22)
       cyoi_provider_client.create_security_group("dns_server", "dns_server", protocol: "udp", ports: (53..53) )
       cyoi_provider_client.create_security_group("bosh", "bosh", [4222, 6868, 25250, 25555, 25777] )

data/lib/bosh-bootstrap/public_stemcell.rb

@@ -0,0 +1,33 @@
+module Bosh::Bootstrap
+  class PublicStemcell
+    attr_reader :size
+
+    def initialize(key, size)
+      @key = key
+      @size = size
+
+      @parsed_version = key.scan(/[\d]*_?[\d]+/).first
+    end
+
+    def name
+      File.basename(@key)
+    end
+
+    def version
+      @parsed_version.gsub('_', '.').to_f
+    end
+
+    def variety
+      name.gsub(/(.tgz)|(bosh-stemcell-)|(#{@parsed_version})/, '').split('-').reject { |c| c.empty? }.join('-')
+    end
+
+    def url
+      "#{PublicStemcells::PUBLIC_STEMCELLS_BASE_URL}/#{@key}"
+    end
+
+    def legacy?
+      @key.include?('legacy')
+    end
+
+  end
+end

data/lib/bosh-bootstrap/public_stemcells.rb

@@ -0,0 +1,66 @@
+require 'httpclient'
+require 'rexml/document'
+
+require 'bosh-bootstrap/public_stemcell'
+
+module Bosh::Bootstrap
+  class PublicStemcells
+    PUBLIC_STEMCELLS_BASE_URL = 'https://bosh-jenkins-artifacts.s3.amazonaws.com'
+
+    def has_stemcell?(stemcell_filename)
+      all.any? { |stemcell| stemcell.name == stemcell_filename }
+    end
+
+    def find(stemcell_filename)
+      all.detect { |stemcell| stemcell.name == stemcell_filename }
+    end
+
+    def all
+      response = self.class.http_client.get(PUBLIC_STEMCELLS_BASE_URL, {'prefix' => 'bosh-stemcell'})
+      doc = REXML::Document.new(response.body)
+      stemcells_tags = parse_document(doc)
+      stemcells = parse_stemcells(stemcells_tags)
+
+      while is_truncated(doc)
+        response = self.class.http_client.get(PUBLIC_STEMCELLS_BASE_URL, {
+          'prefix' => 'bosh-stemcell',
+          'marker' => stemcells_tags.last.get_text('Key').value
+        })
+
+        doc = REXML::Document.new(response.body)
+        stemcells_tags = parse_document(doc)
+        stemcells += parse_stemcells(stemcells_tags)
+      end
+
+      stemcells
+    end
+
+    def recent
+      stemcell_varietes = all.reject(&:legacy?).group_by(&:variety).values
+      stemcell_varietes.map { |stemcells| stemcells.sort_by(&:version).last }
+    end
+
+    private
+
+    def parse_document(doc)
+      REXML::XPath.match(doc, "/ListBucketResult/Contents[Key[text()[not(contains(.,'latest'))]]]")
+    end
+
+    def parse_stemcells(stemcell_tags)
+      stemcell_tags.map do |stemcell_tag|
+        stemcell_key = stemcell_tag.get_text('Key').value
+        stemcell_size = Integer(stemcell_tag.get_text('Size').value)
+
+        PublicStemcell.new(stemcell_key, stemcell_size)
+      end
+    end
+
+    def is_truncated(doc)
+      REXML::XPath.match(doc, "/ListBucketResult/IsTruncated").first.get_text == 'true'
+    end
+
+    def self.http_client
+      @http_client ||= HTTPClient.new
+    end
+  end
+end

data/lib/bosh-bootstrap/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Bootstrap
-    VERSION = "0.13.2"
+    VERSION = "0.14.0"
   end
 end

data/spec/integration/aws/aws_ec2_basic_spec.rb

@@ -12,12 +12,14 @@ describe "AWS deployment using gems and publish stemcells" do
 
   # after { destroy_test_constructs }
 
-  it "creates an EC2 inception/microbosh with the associated resources" do
+  xit "creates an EC2 inception/microbosh with the associated resources" do
     # TODO remove this when finished initial development
     require "fog"
     Fog.mock!
 
     setup_home_dir
+    puts `pwd`
+    puts `tree`
     create_manifest
     cli.deploy
 

data/spec/spec_helper.rb

@@ -39,21 +39,40 @@ def yaml_files_match(filename, expected_filename)
 end
 
 def setup_home_dir
-  home_dir = File.expand_path("../../tmp/home", __FILE__)
   FileUtils.rm_rf(home_dir)
   FileUtils.mkdir_p(home_dir)
   ENV['HOME'] = home_dir
 end
 
+def setup_work_dir
+  FileUtils.mkdir_p(work_dir)
+  FileUtils.chdir(work_dir)
+end
+
+def work_dir
+  File.join(home_dir, "workspace/deployments/microbosh")
+end
+
+def home_dir
+  File.expand_path("../../tmp/home", __FILE__)
+end
+
 # returns the file path to a file
 # in the fake $HOME folder
 def home_file(*path)
   File.join(ENV['HOME'], *path)
 end
 
+# returns the file path to a file
+# in the fake ~/workspace/deployments/microbosh folder
+def work_file(*path)
+  File.join(work_dir, *path)
+end
+
 RSpec.configure do |c|
   c.before(:each) do
     setup_home_dir
+    setup_work_dir
   end
 
   c.color = true

data/spec/unit/commands/delete_spec.rb

@@ -6,8 +6,6 @@ describe Bosh::Bootstrap::Cli::Commands::Delete do
   include StdoutCapture
   include Bosh::Bootstrap::Cli::Helpers
 
-  let(:settings_dir) { File.expand_path("~/.microbosh") }
-
   subject { Bosh::Bootstrap::Cli::Commands::Delete.new }
 
   it "deletes microbosh VM" do
@@ -17,4 +15,4 @@ describe Bosh::Bootstrap::Cli::Commands::Delete do
     subject.should_receive(:sh).with("bundle exec bosh -n micro delete")
     subject.perform
   end
-end
+end

data/spec/unit/commands/deploy_spec.rb

@@ -7,7 +7,7 @@ describe Bosh::Bootstrap::Cli::Commands::Deploy do
   include StdoutCapture
   include Bosh::Bootstrap::Cli::Helpers
 
-  let(:settings_dir) { File.expand_path("~/.microbosh") }
+  let(:settings_dir) { work_dir }
 
   before do
     FileUtils.mkdir_p(@stemcells_dir = File.join(Dir.mktmpdir, "stemcells"))
@@ -45,8 +45,8 @@ describe Bosh::Bootstrap::Cli::Commands::Deploy do
 
       # microbosh_provider & select_public_image_or_download_stemcell
       microbosh_provider = instance_double("Bosh::Bootstrap::MicroboshProviders::AWS")
-      expect(microbosh_provider).to receive(:stemcell).exactly(1).times.and_return("")
-      expect(microbosh_provider).to receive(:stemcell).exactly(1).times.and_return("ami-123456")
+      expect(microbosh_provider).to receive(:stemcell_path).exactly(1).times.and_return("")
+      expect(microbosh_provider).to receive(:stemcell_path).exactly(1).times.and_return("ami-123456")
       expect(cmd).to receive(:microbosh_provider).and_return(microbosh_provider).exactly(3).times
 
       # setup_keypair
@@ -91,8 +91,8 @@ describe Bosh::Bootstrap::Cli::Commands::Deploy do
 
       # microbosh_provider & select_public_image_or_download_stemcell
       microbosh_provider = instance_double("Bosh::Bootstrap::MicroboshProviders::OpenStack")
-      expect(microbosh_provider).to receive(:stemcell).exactly(1).times.and_return("")
-      expect(microbosh_provider).to receive(:stemcell).exactly(1).times.and_return("openstack.tgz")
+      expect(microbosh_provider).to receive(:stemcell_path).exactly(1).times.and_return("")
+      expect(microbosh_provider).to receive(:stemcell_path).exactly(1).times.and_return("openstack.tgz")
       expect(cmd).to receive(:microbosh_provider).and_return(microbosh_provider).exactly(3).times
 
       # setup_keypair

data/spec/unit/commands/ssh_spec.rb

@@ -6,15 +6,15 @@ describe Bosh::Bootstrap::Cli::Commands::SSH do
   include StdoutCapture
   include Bosh::Bootstrap::Cli::Helpers
 
-  let(:settings_dir) { File.expand_path("~/.microbosh") }
+  let(:settings_dir) { work_dir }
 
   subject { Bosh::Bootstrap::Cli::Commands::SSH.new }
 
   it "runs ssh" do
     setting "address.ip", "1.2.3.4"
     setting "key_pair.path", "/path/to/private/key"
-    subject.should_receive(:setup_keypair)
-    subject.should_receive(:sh).with("ssh -i /path/to/private/key vcap@1.2.3.4")
+    expect(subject).to receive(:setup_keypair)
+    expect(subject).to receive(:sh).with("ssh -i /path/to/private/key vcap@1.2.3.4")
     subject.perform
   end
-end
+end

data/spec/unit/key_pair_spec.rb

@@ -3,13 +3,12 @@ require "bosh-bootstrap/key_pair"
 describe Bosh::Bootstrap::KeyPair do
   include Bosh::Bootstrap::Cli::Helpers::Settings
 
-  subject { Bosh::Bootstrap::KeyPair.new(settings_dir, "keyname", "PRIVATE") }
+  subject { Bosh::Bootstrap::KeyPair.new(work_dir, "keyname", "PRIVATE") }
 
   it "creates local private key file" do
-    setup_home_dir
-    expect(File.exists?(home_file(".microbosh", "ssh", "keyname"))).to eq false
+    expect(File.exists?(work_file("ssh", "keyname"))).to eq false
     subject.execute!
-    keyfile = home_file(".microbosh", "ssh", "keyname")
+    keyfile = work_file("ssh", "keyname")
     expect(File.read(keyfile)).to eq "PRIVATE"
   end
 end

data/spec/unit/microbosh_providers/aws_spec.rb

@@ -5,10 +5,9 @@ require "bosh-bootstrap/microbosh_providers/aws"
 describe Bosh::Bootstrap::MicroboshProviders::AWS do
   include Bosh::Bootstrap::Cli::Helpers::Settings
 
-  let(:microbosh_yml) { File.expand_path("~/.microbosh/deployments/micro_bosh.yml")}
-  let(:aws_jenkins_bucket) { "bosh-jenkins-artifacts" }
-  let(:latest_ami_uri) { "http://#{aws_jenkins_bucket}.s3.amazonaws.com/last_successful-bosh-stemcell-aws_ami_us-east-1" }
-  let(:latest_stemcell_uri) { "http://bosh-jenkins-artifacts.s3.amazonaws.com/bosh-stemcell/aws/bosh-stemcell-latest-aws-xen-ubuntu.tgz" }
+  let(:microbosh_yml) { File.expand_path("~/.microbosh/deployments/micro_bosh.yml") }
+  let(:artifacts_base) { "https://bosh-jenkins-artifacts.s3.amazonaws.com" }
+  let(:http_client) { instance_double("HTTPClient") }
 
   it "creates micro_bosh.yml manifest" do
     setting "provider.name", "aws"
@@ -24,7 +23,7 @@ describe Bosh::Bootstrap::MicroboshProviders::AWS do
     subject = Bosh::Bootstrap::MicroboshProviders::AWS.new(microbosh_yml, settings)
 
     subject.create_microbosh_yml(settings)
-    File.should be_exists(microbosh_yml)
+    expect(File).to be_exists(microbosh_yml)
     yaml_files_match(microbosh_yml, spec_asset("microbosh_yml/micro_bosh.aws_ec2.yml"))
   end
 
@@ -43,7 +42,7 @@ describe Bosh::Bootstrap::MicroboshProviders::AWS do
     subject = Bosh::Bootstrap::MicroboshProviders::AWS.new(microbosh_yml, settings)
 
     subject.create_microbosh_yml(settings)
-    File.should be_exists(microbosh_yml)
+    expect(File).to be_exists(microbosh_yml)
     yaml_files_match(microbosh_yml, spec_asset("microbosh_yml/micro_bosh.aws_ec2.us-west-2a.yml"))
   end
 
@@ -52,24 +51,52 @@ describe Bosh::Bootstrap::MicroboshProviders::AWS do
       setting "provider.name", "aws"
     end
 
-    it "is an AMI if us-east-1 target region" do
-      setting "provider.region", "us-east-1"
-      FakeWeb.register_uri(:get, latest_ami_uri, body: "ami-234567")
-
-      subject = Bosh::Bootstrap::MicroboshProviders::AWS.new(microbosh_yml, settings)
-      subject.stemcell.should == "ami-234567"
+    before(:each) do
+      body = <<-XML
+<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>bosh-jenkins-artifacts</Name>
+  <Prefix>bosh-stemcell</Prefix>
+  <Marker/>
+  <MaxKeys>1000</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+  <Contents>
+    <Key>bosh-stemcell/aws/bosh-stemcell-2719-aws-xen-centos-go_agent.tgz</Key>
+    <LastModified>2014-09-22T04:59:16.000Z</LastModified>
+    <ETag>"7621366406eeb0a9d88242a664206cc3"</ETag>
+    <Size>557556059</Size>
+    <StorageClass>STANDARD</StorageClass>
+  </Contents>
+  <Contents>
+    <Key>bosh-stemcell/aws/bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz</Key>
+    <LastModified>2014-09-22T04:59:16.000Z</LastModified>
+    <ETag>"18cb27adc889e71c97e39b1c57f85027"</ETag>
+    <Size>467288141</Size>
+    <StorageClass>STANDARD</StorageClass>
+  </Contents>
+  <Contents>
+    <Key>bosh-stemcell/aws/light-bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz</Key>
+    <LastModified>2014-09-22T04:59:16.000Z</LastModified>
+    <ETag>"18cb27adc889e71c97e39b1c57f85027"</ETag>
+    <Size>467288141</Size>
+    <StorageClass>STANDARD</StorageClass>
+  </Contents>
+</ListBucketResult>
+      XML
+      expect(Bosh::Bootstrap::PublicStemcells).to receive(:http_client).and_return(http_client)
+      expect(http_client).to receive(:get).
+        with(artifacts_base, {'prefix' => 'bosh-stemcell'}).
+        and_return(OpenStruct.new(body: body))
     end
 
-    it "retries to get AMI if initially fails" do
+    it "light stemcell if us-east-1 target region" do
       setting "provider.region", "us-east-1"
-      FakeWeb.register_uri(:get, latest_ami_uri, [
-        { status: 404 },
-        { body: "ami-234567"}
-      ])
 
       subject = Bosh::Bootstrap::MicroboshProviders::AWS.new(microbosh_yml, settings)
-      subject.stemcell.should == ""
-      subject.stemcell.should == "ami-234567"
+
+      latest_stemcell_uri = "#{artifacts_base}/bosh-stemcell/aws/light-bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz"
+      expect(subject).to receive(:sh).with("curl -O '#{latest_stemcell_uri}'")
+      expect(subject.stemcell_path).to match /light-bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz$/
     end
 
     xit "errors if AMI not available and not running within target region" do
@@ -78,10 +105,15 @@ describe Bosh::Bootstrap::MicroboshProviders::AWS do
 
     it "downloads latest stemcell and returns path if running in target AWS region" do
       setting "provider.region", "us-west-2"
-      
+
       subject = Bosh::Bootstrap::MicroboshProviders::AWS.new(microbosh_yml, settings)
-      subject.stub(:sh).with("curl -O '#{latest_stemcell_uri}'")
-      subject.stemcell.should =~ %r{deployments/bosh-stemcell-latest-aws-xen-ubuntu.tgz$}
+
+      latest_stemcell_uri = "#{artifacts_base}/bosh-stemcell/aws/bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz"
+
+      expect(subject).to receive(:sh).with("curl -O '#{latest_stemcell_uri}'")
+      stemcell_path = subject.stemcell_path
+      expect(stemcell_path).to match /bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz$/
+      expect(stemcell_path).to_not match /light-bosh-stemcell-2719-aws-xen-ubuntu-trusty-go_agent.tgz$/
     end
   end
 end

data/spec/unit/microbosh_spec.rb

@@ -3,18 +3,18 @@ require "readwritesettings"
 describe Bosh::Bootstrap::Microbosh do
   include Bosh::Bootstrap::Cli::Helpers::Settings
 
-  let(:path_or_ami) { "/path/to/stemcell.tgz" }
-  let(:base_path) { File.expand_path("~/.microbosh") }
+  let(:stemcell_path) { "/path/to/stemcell.tgz" }
+  let(:base_path) { File.expand_path("/tmp/bootstrap") }
   let(:settings_dir) { base_path }
   let(:microbosh_provider) { instance_double("Bosh::Bootstrap::MicroboshProviders::AWS", create_microbosh_yml: {}) }
   subject { Bosh::Bootstrap::Microbosh.new(base_path, microbosh_provider) }
 
   it "deploys new microbosh" do
     setting "bosh.name", "test-bosh"
-    setting "bosh.stemcell", path_or_ami
-    subject.should_receive(:sh).with("bundle install")
-    subject.should_receive(:sh).with("bundle exec bosh micro deployment test-bosh")
-    subject.should_receive(:sh).with("bundle exec bosh -n micro deploy --update-if-exists #{path_or_ami}")
+    setting "bosh.stemcell_path", stemcell_path
+    expect(subject).to receive(:sh).with("bundle install")
+    expect(subject).to receive(:sh).with("bundle exec bosh micro deployment test-bosh")
+    expect(subject).to receive(:sh).with("bundle exec bosh -n micro deploy --update-if-exists #{stemcell_path}")
     subject.deploy(settings)
   end
 

data/spec/unit/network_providers/aws_spec.rb

@@ -13,15 +13,30 @@ describe Bosh::Bootstrap::NetworkProviders::AWS do
     expect(Bosh::Bootstrap::NetworkProviders.provider_class("aws")).to eq(subject.class)
   end
 
-  it "creates security groups it needs" do
+  it "creates EC2 security groups it needs" do
     expected_groups = [
       ["ssh", "ssh", ports: 22],
       ["dns_server", "dns_server", ports: { protocol: "udp", ports: (53..53) }],
       ["bosh", "bosh", ports: [4222, 6868, 25250, 25555, 25777]]
     ]
     expected_groups.each do |security_group_name, description, ports|
-      expect(cyoi_provider_client).to receive(:create_security_group).with(security_group_name, description, ports)
+      expect(cyoi_provider_client).to receive(:create_security_group).with(security_group_name, description, ports, {})
     end
-    subject.perform
+    subject.perform(settings)
   end
+
+  it "creates VPC security groups it needs" do
+    setting "address.vpc_id", "vpc-id-1234"
+    expected_groups = [
+      ["ssh", "ssh", ports: 22],
+      ["dns_server", "dns_server", ports: { protocol: "udp", ports: (53..53) }],
+      ["bosh", "bosh", ports: [4222, 6868, 25250, 25555, 25777]]
+    ]
+    expected_groups.each do |security_group_name, description, ports|
+      expect(cyoi_provider_client).to receive(:create_security_group).
+        with(security_group_name, description, ports, {vpc_id: "vpc-id-1234"})
+    end
+    subject.perform(settings)
+  end
+
 end

data/spec/unit/network_providers/openstack_spec.rb

@@ -21,6 +21,6 @@ describe Bosh::Bootstrap::NetworkProviders::OpenStack do
     expected_groups.each do |security_group_name, description, ports|
       expect(provider_client).to receive(:create_security_group).with(security_group_name, description, ports)
     end
-    subject.perform
+    subject.perform(settings)
   end
 end

data/spec/unit/network_spec.rb

@@ -6,13 +6,13 @@ describe Bosh::Bootstrap::Network do
   let(:aws_cyoi_provider_client) { instance_double("Cyoi::Providers::Clients::AwsProviderClient") }
 
   it "uses NetworkProvider if available" do
-    network = Bosh::Bootstrap::Network.new("aws", aws_cyoi_provider_client)
+    network = Bosh::Bootstrap::Network.new("aws", aws_cyoi_provider_client, settings)
     expect(aws_cyoi_provider_client).to receive(:create_security_group).exactly(3).times
     network.deploy
   end
 
   it "does nothing if no NetworkProvider for the infrastructure" do
-    network = Bosh::Bootstrap::Network.new("vsphere", nil)
+    network = Bosh::Bootstrap::Network.new("vsphere", nil, settings)
     network.deploy
   end
 end

metadata

@@ -1,139 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: bosh-bootstrap
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 0.14.0
 platform: ruby
 authors:
 - Dr Nic Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-30 00:00:00.000000000 Z
+date: 2014-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cyoi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.11'
 - !ruby/object:Gem::Dependency
   name: readwritesettings
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: redcard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rbvmomi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-fire
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fakeweb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Bootstrap a micro bosh universe from one CLI
@@ -144,9 +158,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - ChangeLog.md
 - Gemfile
 - Guardfile
@@ -177,6 +191,8 @@ files:
 - lib/bosh-bootstrap/network_providers/aws.rb
 - lib/bosh-bootstrap/network_providers/dummy.rb
 - lib/bosh-bootstrap/network_providers/openstack.rb
+- lib/bosh-bootstrap/public_stemcell.rb
+- lib/bosh-bootstrap/public_stemcells.rb
 - lib/bosh-bootstrap/thor_cli.rb
 - lib/bosh-bootstrap/version.rb
 - lib/bosh/cli/commands/bootstrap.rb
@@ -217,17 +233,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: bosh-bootstrap configures and deploys a microbosh deployed on either AWS