bosh-bootstrap 0.8.2 → 0.9.0

data/.gitignore

@@ -1,4 +1,3 @@
-*.gem
 *.rbc
 .bundle
 .config

data/.travis.yml

@@ -1,5 +1,26 @@
 language: ruby
+script: bundle exec rake spec:$SUITE
+bundler_args: "--local"
 rvm:
-  - 1.9.3
-  - rbx-19mode
-  # - ruby-head - generates "Cannot find Syck parser for YAML"
+  - ruby-1.9.3
+  # - rbx-19mode
+  # - ruby-2.0.0 - generates "Cannot find Syck parser for YAML"
+notifications:
+  email:
+    recipients:
+      - drnicwilliams@gmail.com
+    on_success: change
+    on_failure: always
+env:
+  matrix:
+    - SUITE=unit
+    # - SUITE=integration:aws:basic
+    - SUITE=integration:aws:edge
+    # - SUITE=integration:aws:edge_prebuilt
+    # - SUITE=integration:aws:edge_prebuilt_ami
+  global:
+    - secure: "OS3pnXOAVRP0QPDl/Nn/0iBPZbrnn9irFYwEbMJ2fGPsV00u4IYDwoIN5Gzb\nX+cXyiPZB0jdQlbiCMaQEYjYig3fgnDhiLj/MnxkVYtILm+0uPSrX2Zf7jvR\nmxCT5jfTSZoPXQeNi9h8rEqvaqKv4z930xp/Z0QNoZLPWdwY5wk="
+matrix:
+  allow_failures:
+    - env: SUITE=integration:aws:basic
+        

data/ChangeLog.md

@@ -2,6 +2,14 @@
 
 `bosh-bootstrap` is a command line tool that you can run on your laptop and automatically get a microbosh (and an inception VM) deployed on either AWS or OpenStack.
 
+## v0.9
+
+* v0.8 wasn't working for many people; and neither will v0.9; but its a move in the right direction.
+* Moving towards new 1.5.0 version of bosh that hasn't come out yet formally.
+* AWS us-east-1 will use a pre-created AMI. It saves about 10-15 minutes!
+* AWS other regions will use a pre-created stemcell. I haven't tested this well yet.
+* OpenStack support is still broken because you need to create your own stemcells and for that you need a 12.10 inception VM and that work isn't quite done yet.
+
 ## v0.8 (& v0.8.1)
 
 * SSH keys used to access inception VM are now generated and stored within the `~/.bosh_bootstrap/ssh` folder. This fixes many issues that many people were having (their keys had passphrases, their fog_default keypair was old). It also allows a manifest file to be shared between people as it contains the private key contents, and the private key file will be recreated if it is missing.

data/Gemfile

@@ -3,6 +3,8 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in bosh-bootstrap.gemspec
 gemspec
 
-gem "awesome_print"
-gem "rb-fsevent", "~> 0.9.1"
-gem "guard-rspec"
+group :development do
+  gem "awesome_print"
+  gem "rb-fsevent", "~> 0.9.1"
+  gem "guard-rspec"
+end

data/Guardfile

@@ -1,6 +1,6 @@
-guard 'rspec' do
-  watch(%r{^spec/.+_spec\.rb$})
+guard 'rspec', spec_paths: ["spec/unit"] do
+  watch(%r{^spec/unit/(.+_spec)\.rb$})
   watch(%r{^lib/bosh-bootstrap/(.+)\.rb$})    { |m| "spec/unit/#{m[1]}_spec.rb" }
-  watch('spec/spec_helper.rb')  { "spec" }
+  watch('spec/spec_helper.rb')  { "spec/unit" }
 end
 

data/Rakefile

@@ -19,15 +19,31 @@ if defined?(RSpec)
       t.rspec_opts = %w(--format progress --color)
     end
 
-    desc "Run Integration Tests"
-    functional_rspec_task = RSpec::Core::RakeTask.new(:functional) do |t|
-      t.pattern = "spec/functional/**/*_spec.rb"
-      t.rspec_opts = %w(--format progress --color)
+    namespace :integration do
+      namespace :aws do
+        jobs = Dir["spec/integration/aws/*_spec.rb"].map {|f| File.basename(f).gsub(/aws_(.*)_spec.rb/, '\1')}
+        jobs.each do |job|
+          desc "Run AWS '#{job}' Integration Test"
+          RSpec::Core::RakeTask.new(job.to_sym) do |t|
+            t.pattern = "spec/integration/aws/aws_#{job}_spec.rb"
+            t.rspec_opts = %w(--format progress --color)
+          end
+        end
+      end
+
+      desc "Run AWS Integration Tests"
+      RSpec::Core::RakeTask.new(:aws) do |t|
+        t.pattern = "spec/integration/aws/*_spec.rb"
+        t.rspec_opts = %w(--format progress --color)
+      end
     end
+
+    desc "Run all Integration Tests"
+    task :integration => %w[spec:integration:aws]
   end
 
   desc "Install dependencies and run tests"
-  task :spec => %w(spec:unit spec:functional)
+  task :spec => %w(spec:unit spec:integration)
 end
 
 task :default => :spec

data/lib/bosh/providers/aws.rb

@@ -123,33 +123,6 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     true
   end
 
-  # Any of the following +port_defn+ can be used:
-  # {
-  #   ssh: 22,
-  #   http: { ports: (80..82) },
-  #   mosh: { protocol: "udp", ports: (60000..60050) }
-  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_range: "196.212.12.34/32" }
-  # }
-  # In this example, 
-  #  * TCP 22 will be opened for ssh from any ip_range,
-  #  * TCP ports 80, 81, 82 for http from any ip_range,
-  #  * UDP 60000 -> 60050 for mosh from any ip_range and
-  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
-  def extract_port_definition(port_defn)
-    protocol = "tcp"
-    ip_range = "0.0.0.0/0"
-    if port_defn.is_a? Integer
-      port_range = (port_defn..port_defn)
-    elsif port_defn.is_a? Range
-      port_range = port_defn
-    elsif port_defn.is_a? Hash
-      protocol = port_defn[:protocol] if port_defn[:protocol]
-      port_range = port_defn[:ports]  if port_defn[:ports]
-      ip_range = port_defn[:ip_range] if port_defn[:ip_range]
-    end
-    [protocol, port_range, ip_range]
-  end
-
   def port_open?(ip_permissions, port_range, protocol, ip_range)
     ip_permissions && ip_permissions.find do |ip| 
       ip["ipProtocol"] == protocol \
@@ -181,7 +154,34 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     volume.server = server
   end
 
+  # Ubuntu 12.10 64bit (EBS) - Quantal
+  def quantal_image_id(region)
+    # http://cloud-images.ubuntu.com/quantal/current/
+    image_id = case region.to_s
+    when 'ap-northeast-1'
+      'ami-ccf270cd'
+    when 'ap-southeast-1'
+      'ami-16e8a444'
+    when 'ap-southeast-2'
+      'ami-5af36360'
+    when 'eu-west-1'
+      'ami-789c890c'
+    when 'sa-east-1'
+      'ami-35b36928'
+    when 'us-east-1'
+      'ami-1c80e475'
+    when 'us-west-1'
+      'ami-28567a6d'
+    when 'us-west-2'
+      'ami-5822b668'
+    end
+    image_id || raise("Please add Ubuntu 12.10 64bit (EBS) AMI image id to aws.rb#image_id method for region '#{region}'")
+  end
+
   def bootstrap(new_attributes = {})
+    if new_attributes.delete(:quantal)
+      new_attributes[:image_id] ||= quantal_image_id(fog_compute.region)
+    end
     vpc = new_attributes[:subnet_id]
 
     server = fog_compute.servers.new(new_attributes)
@@ -215,4 +215,44 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     server
   end
 
+  def servers_with_sg(sg_name)
+    inception_sg = fog_compute.security_groups.find {|sg| sg.name == sg_name }
+    if inception_sg
+      fog_compute.servers.select {|s| s.security_group_ids.include? inception_sg.group_id }
+    else
+      $stderr.puts "no security group #{sg_name} was found"
+      []
+    end
+  end
+
+  def delete_security_group_and_servers(sg_name)
+    sg = fog_compute.security_groups.find {|sg| sg.name == sg_name }
+    if sg
+      fog_compute.servers.select {|s| s.security_group_ids.include? sg.group_id }.each do |server|
+        puts "Destroying server #{server.id}..."
+        server.destroy
+      end
+      begin
+        puts "Destroying security group #{sg.name}..."
+        sg.destroy
+      rescue Fog::Compute::AWS::Error => e
+        $stderr.puts e
+      end
+    end
+  end
+
+  def delete_key_pair(kp_name)
+    if kp = fog_compute.key_pairs.find {|kp| kp.name == kp_name}
+      puts "Deleting key pair #{kp.name}..."
+      kp.destroy
+    end
+  end
+
+  # Destroy all IP addresses that aren't bound to a server
+  def cleanup_unused_ip_addresses
+    fog_compute.addresses.each do |a|
+      puts "Deleting IP address #{a.public_ip}..."
+      a.destroy unless a.server
+    end
+  end
 end

data/lib/bosh/providers/base_provider.rb

@@ -18,4 +18,31 @@ class Bosh::Providers::BaseProvider
       fog_key_pair.destroy
     end
   end
+
+  # Any of the following +port_defn+ can be used:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_range: "196.212.12.34/32" }
+  # }
+  # In this example,
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
+  def extract_port_definition(port_defn)
+    protocol = "tcp"
+    ip_range = "0.0.0.0/0"
+    if port_defn.is_a? Integer
+      port_range = (port_defn..port_defn)
+    elsif port_defn.is_a? Range
+      port_range = port_defn
+    elsif port_defn.is_a? Hash
+      protocol = port_defn[:protocol] if port_defn[:protocol]
+      port_range = port_defn[:ports]  if port_defn[:ports]
+      ip_range = port_defn[:ip_range] if port_defn[:ip_range]
+    end
+    [protocol, port_range, ip_range]
+  end
 end

data/lib/bosh/providers/openstack.rb

@@ -37,10 +37,11 @@ class Bosh::Providers::OpenStack < Bosh::Providers::BaseProvider
     end
     ip_permissions = sg.rules
     ports_opened = 0
-    ports.each do |name, port|
-      unless port_open?(ip_permissions, port)
-        sg.create_security_group_rule(port, port)
-        puts " -> opened #{name} port #{port}"
+    ports.each do |name, port_defn|
+      (protocol, port_range, ip_range) = extract_port_definition(port_defn)
+      unless port_open?(ip_permissions, port_range, protocol, ip_range)
+        sg.create_security_group_rule(port_range.min, port_range.max, protocol, ip_range)
+        puts " -> opened #{name} ports #{protocol.upcase} #{port_range.min}..#{port_range.max} from IP range #{ip_range}"
         ports_opened += 1
       end
     end
@@ -48,8 +49,13 @@ class Bosh::Providers::OpenStack < Bosh::Providers::BaseProvider
     true
   end
 
-  def port_open?(ip_permissions, port)
-    ip_permissions && ip_permissions.find {|ip| ip["from_port"] <= port && ip["to_port"] >= port }
+  def port_open?(ip_permissions, port_range, protocol, ip_range)
+    ip_permissions && ip_permissions.find do |ip|
+      ip["ip_protocol"] == protocol \
+      && ip["ip_range"].detect { |range| range["cidr"] == ip_range } \
+      && ip["from_port"] <= port_range.min \
+      && ip["to_port"] >= port_range.max
+    end
   end
 
   def find_server_device(server, device)
@@ -66,4 +72,8 @@ class Bosh::Providers::OpenStack < Bosh::Providers::BaseProvider
     volume.attach(server.id, device)
     volume.wait_for { volume.status == 'in-use' }
   end
+
+  def delete_security_group_and_servers(sg_name)
+    raise "not implemented yet"
+  end
 end