rubber 3.1.0 → 3.2.0

data/lib/rubber/cloud/base.rb

@@ -10,7 +10,7 @@ module Rubber
         @capistrano = capistrano
       end
 
-      def before_create_instance(instance_alias, role_names)
+      def before_create_instance(instance)
         # No-op by default.
       end
 
@@ -143,6 +143,10 @@ module Rubber
             source_ips = rule['source_ips']
 
             if protocol && source_ips
+              if source_ips.respond_to?(:split)
+                source_ips = source_ips.split(",").map(&:strip)
+              end
+
               source_ips.each do |source|
                 if from_port && to_port
                   if from_port != to_port
@@ -184,6 +188,10 @@ exit 0
         capistrano.run_script('setup_firewall_rules', script, :hosts => instance.external_ip)
       end
 
+      def setup_vpc
+        # No-op by default.
+      end
+
       def describe_security_groups(group_name=nil)
         rules = capistrano.capture("iptables -S INPUT", :hosts => rubber_env.rubber_instances.collect(&:external_ip)).strip.split("\r\n")
         scoped_rules = rules.select { |r| r =~ /dport/ }
@@ -223,4 +231,4 @@ exit 0
     end
 
   end
-end
+end

data/lib/rubber/cloud/digital_ocean.rb

@@ -1,3 +1,4 @@
+require 'ext/fog/compute/digital_ocean_v2'
 require 'rubber/cloud/fog'
 
 module Rubber
@@ -8,8 +9,8 @@ module Rubber
       def initialize(env, capistrano)
         compute_credentials = {
           :provider => 'DigitalOcean',
-          :digitalocean_api_key => env.api_key,
-          :digitalocean_client_id => env.client_key
+          :version => 'v2',
+          :digitalocean_token => env.digital_ocean_token,
         }
 
         if env.cloud_providers && env.cloud_providers.aws
@@ -29,17 +30,13 @@ module Rubber
         super(env, capistrano)
       end
 
-      # As of October 2014 Digital Ocean supports private networking in
-      # New York 2 (id 4), New York 3 (id 8), Amsterdam 2 (id 5), Amsterdam 3 (id 9), Singapore 1 (id 6) and London 1 (id 7)
-      REGIONS_WITH_PRIVATE_NETWORKING = [4, 5, 6, 7, 8, 9]
-
       def create_instance(instance_alias, image_name, image_type, security_groups, availability_zone, region, fog_options={})
-        do_region = compute_provider.regions.find { |r| r.name == region }
+        do_region = compute_provider.regions.find { |r| [r.name, r.slug].include?(region) }
         if do_region.nil?
           raise "Invalid region for DigitalOcean: #{region}"
         end
 
-        if env.private_networking && ! REGIONS_WITH_PRIVATE_NETWORKING.include?(do_region.id)
+        if env.private_networking && ! do_region.features.include?("private_networking")
           raise "Private networking is enabled, but region #{region} does not support it"
         end
 
@@ -48,11 +45,17 @@ module Rubber
           raise "Invalid image name for DigitalOcean: #{image_name}"
         end
 
-        flavor = compute_provider.flavors.find { |f| f.name == image_type }
+        # Downcase image_type for backward compatability with v1
+        flavor = compute_provider.flavors.find { |f| f.slug == image_type.downcase }
+
         if flavor.nil?
           raise "Invalid image type for DigitalOcean: #{image_type}"
         end
 
+        if env.key_name.nil?
+          raise 'missing key_name for DigitalOcean'
+        end
+
         # Check if the SSH key has been added to DigitalOcean yet.
         # TODO (nirvdrum 03/23/13): DigitalOcean has an API for getting a single SSH key, but it hasn't been added to fog yet.  We should add it.
         ssh_key = compute_provider.list_ssh_keys.body['ssh_keys'].find { |key| key['name'] == env.key_name }
@@ -75,13 +78,15 @@ module Rubber
         end
 
         response = compute_provider.servers.create({:name => "#{Rubber.env}-#{instance_alias}",
-                                                   :image_id => image.id,
-                                                   :flavor_id => flavor.id,
-                                                   :region_id => do_region.id,
-                                                   :ssh_key_ids => [ssh_key['id']],
-                                                   :private_networking => (env.private_networking.to_s.downcase == 'true')}.
-                                                   merge(Rubber::Util.symbolize_keys(fog_options))
-        )
+                                                    :image => image.slug,
+                                                    :size => flavor.slug,
+                                                    :flavor => flavor.slug,
+                                                    :region => do_region.slug,
+                                                    :ssh_keys => [ssh_key['id']],
+                                                    :private_networking => (env.private_networking.to_s.downcase == 'true')
+                                                   }
+                                                    .merge(Rubber::Util.symbolize_keys(fog_options))
+                                                  )
 
         response.id
       end
@@ -99,11 +104,28 @@ module Rubber
         response.each do |item|
           instance = {}
           instance[:id] = item.id
-          instance[:state] = item.state
-          instance[:type] = item.flavor_id
-          instance[:external_ip] = item.public_ip_address
-          instance[:internal_ip] = item.private_ip_address || item.public_ip_address
-          instance[:region_id] = item.region_id
+          instance[:state] = item.status
+          instance[:type] = item.size_slug
+
+          public_networking_info = item.networks['v4'].find do |n|
+            n['type'] == 'public'
+          end
+
+          if public_networking_info
+            instance[:external_ip] = public_networking_info['ip_address']
+          end
+
+          private_networking_info = item.networks['v4'].find do |n|
+            n['type'] == 'private'
+          end
+
+          if private_networking_info
+            instance[:internal_ip] = private_networking_info['ip_address']
+          elsif public_networking_info
+            instance[:internal_ip] = public_networking_info['ip_address']
+          end
+
+          instance[:region_id] = item.region
           instance[:provider] = 'digital_ocean'
           instance[:platform] = Rubber::Platforms::LINUX
           instances << instance
@@ -115,6 +137,25 @@ module Rubber
       def active_state
         'active'
       end
+
+      def destroy_instance(instance_id)
+        # The Digital Ocean API will return a 422 if we attempt to destroy an
+        # instance that's in the middle of booting up, so wait until it's
+        # in a non-"new" state
+        print 'Waiting for non-new instance state'
+
+        loop do
+          instance = describe_instances(instance_id).first
+
+          print '.'
+
+          break unless instance[:state] == 'new'
+
+          sleep 1
+        end
+
+        response = compute_provider.servers.get(instance_id).delete()
+      end
     end
   end
 end

data/lib/rubber/cloud/vagrant.rb

@@ -12,9 +12,9 @@ module Rubber
         'saved'
       end
 
-      def before_create_instance(instance_alias, role_names)
+      def before_create_instance(instance)
         unless ENV.has_key?('RUN_FROM_VAGRANT')
-          capistrano.fatal "Since you are using the 'vagrant' provider, you must create instances by running `vagrant up #{instance_alias}`."
+          capistrano.fatal "Since you are using the 'vagrant' provider, you must create instances by running `vagrant up #{instance.instance_alias}`."
         end
       end
 
@@ -61,4 +61,4 @@ module Rubber
       end
     end
   end
-end
+end

data/lib/rubber/instance.rb

@@ -18,7 +18,7 @@ module Rubber
         @opts = opts
       
         @items = {}
-        @artifacts = {'volumes' => {}, 'static_ips' => {}}
+        @artifacts = {'volumes' => {}, 'static_ips' => {}, 'vpc' => {}}
 
         @filters = Rubber::Util::parse_aliases(ENV['FILTER'])
         @filters, @filters_negated = @filters.partition {|f| f !~ /^-/ }
@@ -196,7 +196,10 @@ module Rubber
     # The configuration for a single instance
     class InstanceItem
       UBUNTU_OS_VERSION_CMD = 'lsb_release -sr'.freeze
-      VARIABLES_TO_OMIT_IN_SERIALIZATION = ['@capistrano', '@os_version']
+      VARIABLES_TO_OMIT_IN_SERIALIZATION = [
+        '@capistrano', '@os_version', '@subnet_id', '@vpc_id',
+        '@vpc_cidr'
+      ]
 
       attr_reader :name, :domain, :instance_id, :image_type, :image_id, :security_groups
       attr_accessor :roles, :zone
@@ -206,6 +209,11 @@ module Rubber
       attr_accessor :spot_instance_request_id
       attr_accessor :provider, :platform
       attr_accessor :capistrano
+      attr_accessor :vpc_id
+      attr_accessor :network # more generic term for vpc_alias
+      attr_accessor :vpc_cidr
+      attr_accessor :subnet_id
+      attr_accessor :gateway
 
       def initialize(name, domain, roles, instance_id, image_type, image_id, security_group_list=[])
         @name = name
@@ -227,7 +235,7 @@ module Rubber
         end
         return item
       end
-      
+
       def to_hash
         hash = {}
         instance_variables.each do |iv|
@@ -240,11 +248,11 @@ module Rubber
         end
         return hash
       end
-      
+
       def <=>(rhs)
         name <=> rhs.name
       end
-      
+
       def full_name
         "#{@name}.#{@domain}"
       end

data/lib/rubber/recipes/rubber/instances.rb

@@ -262,20 +262,25 @@ namespace :rubber do
   def create_instance(instance_alias, instance_roles, create_spot_instance)
     role_names = instance_roles.collect{|x| x.name}
     env = rubber_cfg.environment.bind(role_names, instance_alias)
+    cloud_env = env.cloud_providers[env.cloud_provider]
 
-    monitor.synchronize do
-      cloud.before_create_instance(instance_alias, role_names)
-    end
+    availability_zone = cloud_env.availability_zone
 
     security_groups = get_assigned_security_groups(instance_alias, role_names)
 
-    cloud_env = env.cloud_providers[env.cloud_provider]
     ami = cloud_env.image_id
     ami_type = cloud_env.image_type
-    availability_zone = cloud_env.availability_zone
     region = cloud_env.region
     fog_options = cloud_env.fog_options || {}
 
+    instance_item = Rubber::Configuration::InstanceItem.new(instance_alias, env.domain, instance_roles, nil, ami_type, ami, security_groups)
+
+    instance_item.zone = availability_zone
+
+    monitor.synchronize do
+      cloud.before_create_instance(instance_item)
+    end
+
     create_spot_instance ||= cloud_env.spot_instance
 
     if create_spot_instance
@@ -308,20 +313,54 @@ namespace :rubber do
     end
 
     if !create_spot_instance || (create_spot_instance && max_wait_time < 0)
-      logger.info "Creating instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || region || 'Default'}"
-      instance_id = cloud.create_instance(instance_alias, ami, ami_type, security_groups, availability_zone, region, fog_options)
+      sg_str = security_groups.join(',') rescue 'Default'
+      az_str = availability_zone || region || 'Default'
+      vpc_str = instance_item.vpc_id || 'No VPC'
+
+      logger.info "Creating instance #{ami}/#{ami_type}/#{sg_str}/#{az_str}/#{vpc_str}"
+
+      if instance_item.vpc_id
+        fog_options[:vpc_id] = instance_item.vpc_id
+        fog_options[:subnet_id] = instance_item.subnet_id
+        fog_options[:associate_public_ip] = (instance_item.gateway == 'public')
+      end
     end
 
+    # Security Groups are handled in the after_create_instance callback of the
+    # Vpc cloud provider, so pass an empty array here to make sure it isn't
+    # assigned to any other default groups that might be floating around.
+    instance_id = cloud.create_instance(
+      instance_alias,
+      ami,
+      ami_type,
+      fog_options[:vpc_id] ? security_groups : [],
+      availability_zone,
+      region,
+      fog_options
+    )
+
     logger.info "Instance #{instance_alias} created: #{instance_id}"
 
-    instance_item = Rubber::Configuration::InstanceItem.new(instance_alias, env.domain, instance_roles, instance_id, ami_type, ami, security_groups)
-    instance_item.spot_instance_request_id = request_id if create_spot_instance
-    instance_item.capistrano = self
-    rubber_instances.add(instance_item)
+    # Recreate the InstanceItem now that we have an instance_id
+    created_instance_item = Rubber::Configuration::InstanceItem.new(
+      instance_alias,
+      env.domain,
+      instance_roles,
+      instance_id,
+      ami_type,
+      ami,
+      security_groups
+    )
+    created_instance_item.vpc_id = instance_item.vpc_id
+    created_instance_item.network = instance_item.network
+    created_instance_item.spot_instance_request_id = request_id if create_spot_instance
+    created_instance_item.capistrano = self
+    created_instance_item.gateway = instance_item.gateway
+    rubber_instances.add(created_instance_item)
     rubber_instances.save()
 
     monitor.synchronize do
-      cloud.after_create_instance(instance_item)
+      cloud.after_create_instance(created_instance_item)
     end
   end
 
@@ -370,16 +409,28 @@ namespace :rubber do
       rubber_instances.save()
 
       if instance_item.linux?
-        # weird cap/netssh bug, sometimes just hangs forever on initial connect, so force a timeout
         begin
+          # davebenvenuti: Sometimes with VPC subnets, we see an IOError
+          # (connection refused) while the instance is booting, so give it some
+          # time.  It would be preferable to catch the exception and retry, but
+          # I couldn't figure out a good way to do that.
+          sleep 10 if instance_item.network
+
+          # weird cap/netssh bug, sometimes just hangs forever on initial connect, so force a timeout
           Timeout::timeout(env.enable_root_login_timeout || 30) do
             puts 'Trying to enable root login'
 
             # turn back on root ssh access if we are using root as the capistrano user for connecting
-            enable_root_ssh(instance_item.external_ip, fetch(:initial_ssh_user, 'ubuntu')) if user == 'root'
+            if Rubber::Util.is_instance_id?(instance_item.gateway)
+              ip = instance_item.internal_ip
+            else
+              ip = instance_item.external_ip
+            end
+
+            enable_root_ssh(ip, fetch(:initial_ssh_user, 'ubuntu')) if user == 'root'
 
             # force a connection so if above isn't enabled we still timeout if initial connection hangs
-            direct_connection(instance_item.external_ip) do
+            direct_connection(ip) do
               run "echo"
             end
           end
@@ -625,6 +676,7 @@ namespace :rubber do
             when /#{instance_item.full_name}/; ''
             when /#{instance_item.external_host}/; ''
             when /#{instance_item.external_ip}/; ''
+            when /#{instance_item.internal_ip}/; ''
             else line;
           end
           out << line
@@ -650,5 +702,5 @@ namespace :rubber do
     end if rubber_env.role_dependencies
     return deps
   end
-  
 end
+

data/lib/rubber/recipes/rubber/setup.rb

@@ -171,7 +171,12 @@ namespace :rubber do
         end
 
         hosts_data.compact.each do |host_name|
-          local_hosts << ic.external_ip.ljust(18) << host_name << "\n"
+          # Private instances must be connected to via an ssh gateway
+          if Rubber::Util.is_instance_id?(ic.gateway)
+            local_hosts << ic.internal_ip.ljust(18) << host_name << "\n"
+          else
+            local_hosts << ic.external_ip.ljust(18) << host_name << "\n"
+          end
         end
 
       else # non-Windows OS
@@ -189,7 +194,12 @@ namespace :rubber do
           end
         end
 
-        local_hosts << ic.external_ip << ' ' << hosts_data.compact.join(' ') << "\n"
+        # Private instances must be connected to via an ssh gateway
+        if Rubber::Util.is_instance_id?(ic.gateway)
+          local_hosts << ic.internal_ip << ' ' << hosts_data.compact.join(' ') << "\n"
+        else
+          local_hosts << ic.external_ip << ' ' << hosts_data.compact.join(' ') << "\n"
+        end
       end
     end
 

data/lib/rubber/recipes/rubber/vpcs.rb

@@ -0,0 +1,92 @@
+require 'rubber/environment'
+
+namespace :rubber do
+  desc <<-DESC
+    Sets up the VPC identified by vpc_alias
+  DESC
+  required_task :create_vpc do
+    vpc_alias = get_env("ALIAS", "VPC Alias: ", true, Rubber.config.cloud_providers['aws'].vpc_alias)
+    vpc_cidr = get_env("CIDR", "CIDR Block (eg: 10.0.0.0/16): ", true, Rubber.config.cloud_providers['aws'].vpc_cidr)
+
+    cloud.setup_vpc(vpc_alias, vpc_cidr)
+  end
+
+  desc <<-DESC
+    List all VPCs (not just Rubber-created)
+  DESC
+  required_task :describe_vpcs do
+    vpcs = cloud.describe_vpcs
+
+    if vpcs
+      puts "VPCs:"
+
+      vpcs.each do |vpc|
+        puts "\t#{vpc[:id]}\t#{vpc[:name]}\t#{vpc[:rubber_alias]}"
+        vpc[:subnets].each do |subnet|
+          puts "\t\t#{subnet[:id]}\t#{subnet[:rubber_alias]}\t#{subnet[:cidr_block]}\t#{subnet[:public] ? 'public' : 'private'}"
+        end
+      end
+    else
+      puts "No VPCs found"
+    end
+  end
+
+  desc <<-DESC
+    Configure a new subnet on the configured VPC
+  DESC
+  required_task :create_vpc_subnet do
+    vpc_alias = get_env("ALIAS", "VPC Alias: ", true, Rubber.config.cloud_providers['aws'].vpc_alias)
+    cidr = get_env("CIDR", "CIDR Block (eg: 10.0.0.0/24): ", true)
+    gateway = get_env("GATEWAY", "Gateway (\"public\" or NAT instance alias): ", true)
+    zone = get_env("ZONE", "Availability Zone: ", true, Rubber.config.cloud_providers['aws'].availability_zone)
+    name = get_env("NAME", "Subnet name: ", true)
+
+    vpc_id = cloud.compute_provider.vpcs.all('tag:RubberVpcAlias' => vpc_alias).first.id
+
+    unless gateway == 'public'
+      gateway = Rubber.config.rubber_instances.find { |i|
+        i.name == gateway
+      }.instance_id
+    end
+
+    private_nic = {
+      subnet_cidr: subnet_cidr,
+      gateway: gateway
+    }
+
+    cloud.setup_vpc_subnet(vpc_id, vpc_alias, private_nic, zone, name)
+  end
+
+  desc <<-DESC
+    Destroy any VPC with a Rubber-defined Alias
+  DESC
+  required_task :destroy_vpc do
+    vpc_alias = get_env("ALIAS", "VPC Alias: ", true, Rubber.config.cloud_providers['aws'].vpc_alias)
+    value = Capistrano::CLI.ui.ask("Are you sure you want to destroy vpc #{vpc_alias} [yes/NO]?: ")
+
+    if value == 'yes'
+      cloud.destroy_vpc(vpc_alias)
+    else
+      fatal "aborted", 0
+    end
+  end
+
+  desc <<-DESC
+    Refresh the public gateway on a public subnet, or configure a new NAT instance on a private subnet.
+    This currently does not support switching a public subnet to private or vice versa.
+  DESC
+  required_task :update_vpc_gateway do
+    vpc_alias = get_env("ALIAS", "VPC Alias: ", true, Rubber.config.cloud_providers['aws'].vpc_alias)
+    zone = get_env("ZONE", "Availability Zone: ", true, Rubber.config.cloud_providers['aws'].availability_zone)
+    gateway = get_env("GATEWAY", "Gateway (\"public\" or NAT instance alias): ", true)
+
+    unless gateway == 'public'
+      gateway = Rubber.config.rubber_instances.find { |i|
+        i.name == gateway
+      }.instance_id
+    end
+
+    cloud.update_vpc_gateway(vpc_alias, zone, gateway)
+  end
+end
+