rubber 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 072f73ae0dcfe93c27f6c6f63a247df3098be7e4
-  data.tar.gz: f9d4b61bdf4284d7a0d31b9c92a717489e203120
+  metadata.gz: a078638f1d84a5dda8e80d2276dac5c6351bea83
+  data.tar.gz: 54de9550eefdde6a4529336615d37f6eba5860be
 SHA512:
-  metadata.gz: 66d723ae276aaf267270cdac886fc7204de7b64fac814a4dced32810e71b80367b6af1fad91e3bfce5971981f8e98da3a18e094b72cab60004416c8da13b766a
-  data.tar.gz: c37d00455df11c96ecce16c08404296e0bba3cb3691e110380166590ec29477e3a683d42307736ebf29bbbe7e46c22b8642d92e644d9a1a311c6099b9288efc0
+  metadata.gz: 689ed2c04caacc4448c02da2ed3bf1605de897709bc3391d393a611d41ce626165f6cb3149975aad7ba26f689936aca98a01989bc81f5dbb1dc7e23fe7f13202
+  data.tar.gz: a6f0cce4b68dcce1a2473806211c5953e6f0910defa0a2b3a236b4e77352b4abdae978b6012228329064af799349ceea22e10c2428cb8fc827aef5c14d994c27

data/CHANGELOG

@@ -1,3 +1,18 @@
+2.4.0 (05/24/2013)
+------------------
+
+New Features:
+============
+
+[core] DigitalOcean is now supported as a cloud provider
+[core] Added a "generic" cloud provider for connecting to dedicated hardware, VMs, and otherwise unsupported cloud providers
+
+
+Bug Fixes:
+=========
+
+[core] Trigger rubber:config after deploy:update_code so machines with both a DB and the asset pipeline can bootstrap properly <dc5d0b2>
+
 2.3.1 (05/12/2013)
 ------------------
 

data/lib/rubber/cloud.rb

@@ -5,7 +5,7 @@ module Rubber
 
     def self.get_provider(provider, env, capistrano)
       require "rubber/cloud/#{provider}"
-      clazz = Rubber::Cloud.const_get(provider.capitalize)
+      clazz = Rubber::Cloud.const_get(Rubber::Util.camelcase(provider))
       provider_env = env.cloud_providers[provider]
       return clazz.new(provider_env, capistrano)
     end

data/lib/rubber/cloud/aws.rb

@@ -8,24 +8,81 @@ module Rubber
       
       def initialize(env, capistrano)
         
-        credentials = {
-            :aws_access_key_id => env.access_key,
-            :aws_secret_access_key => env.secret_access_key
+        compute_credentials = {
+          :aws_access_key_id => env.access_key,
+          :aws_secret_access_key => env.secret_access_key
         }
+
+        storage_credentials = {
+          :provider => 'AWS',
+          :aws_access_key_id => env.access_key,
+          :aws_secret_access_key => env.secret_access_key
+        }
+
+        @table_store = ::Fog::AWS::SimpleDB.new(compute_credentials)
         
-        @table_store = ::Fog::AWS::SimpleDB.new(credentials)
-        
-        credentials[:region] = env.region
-        @elb = ::Fog::AWS::ELB.new(credentials)
-        
-        credentials[:provider] = 'AWS'
-        env['credentials'] = credentials
+        compute_credentials[:region] = env.region
+        @elb = ::Fog::AWS::ELB.new(compute_credentials)
+
+        compute_credentials[:provider] = 'AWS' # We need to set the provider after the SimpleDB init because it fails if the provider value is specified.
+
+        env['compute_credentials'] = compute_credentials
+        env['storage_credentials'] = storage_credentials
         super(env, capistrano)
       end
       
       def table_store(table_key)
         return Rubber::Cloud::AwsTableStore.new(@table_store, table_key)  
       end
+
+      def describe_instances(instance_id=nil)
+        instances = []
+        opts = {}
+        opts["instance-id"] = instance_id if instance_id
+
+        response = @compute_provider.servers.all(opts)
+        response.each do |item|
+          instance = {}
+          instance[:id] = item.id
+          instance[:type] = item.flavor_id
+          instance[:external_host] = item.dns_name
+          instance[:external_ip] = item.public_ip_address
+          instance[:internal_host] = item.private_dns_name
+          instance[:internal_ip] = item.private_ip_address
+          instance[:state] = item.state
+          instance[:zone] = item.availability_zone
+          instance[:provider] = 'aws'
+          instance[:platform] = item.platform || 'linux'
+          instance[:root_device_type] = item.root_device_type
+          instances << instance
+        end
+
+        return instances
+      end
+
+      def active_state
+        'running'
+      end
+
+      def before_create_instance(instance_alias, role_names)
+        setup_security_groups(instance_alias, role_names)
+      end
+
+      def after_create_instance(instance)
+        # Sometimes tag creation will fail, indicating that the instance doesn't exist yet even though it does.  It seems to
+        # be a propagation delay on Amazon's end, so the best we can do is wait and try again.
+        Rubber::Util.retry_on_failure(StandardError, :retry_sleep => 0.5, :retry_count => 100) do
+          Rubber::Tag::update_instance_tags(instance.name)
+        end
+      end
+
+      def after_refresh_instance(instance)
+        # Sometimes tag creation will fail, indicating that the instance doesn't exist yet even though it does.  It seems to
+        # be a propagation delay on Amazon's end, so the best we can do is wait and try again.
+        Rubber::Util.retry_on_failure(StandardError, :retry_sleep => 0.5, :retry_count => 100) do
+          Rubber::Tag::update_instance_tags(instance.name)
+        end
+      end
       
       def create_image(image_name)
 
@@ -118,6 +175,315 @@ module Rubber
         return lbs
       end
 
+      def describe_availability_zones
+        zones = []
+        response = @compute_provider.describe_availability_zones()
+        items = response.body["availabilityZoneInfo"]
+        items.each do |item|
+          zone = {}
+          zone[:name] = item["zoneName"]
+          zone[:state] =item["zoneState"]
+          zones << zone
+        end
+        return zones
+      end
+
+      def create_spot_instance_request(spot_price, ami, ami_type, security_groups, availability_zone)
+        response = @compute_provider.spot_requests.create(:price => spot_price,
+                                                          :image_id => ami,
+                                                          :flavor_id => ami_type,
+                                                          :groups => security_groups,
+                                                          :availability_zone => availability_zone,
+                                                          :key_name => env.key_name)
+        request_id = response.id
+        return request_id
+      end
+
+      def describe_spot_instance_requests(request_id=nil)
+        requests = []
+        opts = {}
+        opts["spot-instance-request-id"] = request_id if request_id
+        response = @compute_provider.spot_requests.all(opts)
+        response.each do |item|
+          request = {}
+          request[:id] = item.id
+          request[:spot_price] = item.price
+          request[:state] = item.state
+          request[:created_at] = item.created_at
+          request[:type] = item.flavor_id
+          request[:image_id] = item.image_id
+          request[:instance_id] = item.instance_id
+          requests << request
+        end
+        return requests
+      end
+
+      def setup_security_groups(host=nil, roles=[])
+        rubber_cfg = Rubber::Configuration.get_configuration(Rubber.env)
+        scoped_env = rubber_cfg.environment.bind(roles, host)
+        security_group_defns = Hash[scoped_env.security_groups.to_a]
+
+        if scoped_env.auto_security_groups
+          sghosts = (scoped_env.rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
+          sgroles = (scoped_env.rubber_instances.all_roles + roles).uniq.compact
+          security_group_defns = inject_auto_security_groups(security_group_defns, sghosts, sgroles)
+        end
+
+        sync_security_groups(security_group_defns)
+      end
+
+      def describe_security_groups(group_name=nil)
+        groups = []
+
+        opts = {}
+        opts["group-name"] = group_name if group_name
+        response = @compute_provider.security_groups.all(opts)
+
+        response.each do |item|
+          group = {}
+          group[:name] = item.name
+          group[:description] = item.description
+
+          item.ip_permissions.each do |ip_item|
+            group[:permissions] ||= []
+            rule = {}
+
+            rule[:protocol] = ip_item["ipProtocol"]
+            rule[:from_port] = ip_item["fromPort"]
+            rule[:to_port] = ip_item["toPort"]
+
+            ip_item["groups"].each do |rule_group|
+              rule[:source_groups] ||= []
+              source_group = {}
+              source_group[:account] = rule_group["userId"]
+              source_group[:name] = rule_group["groupName"]
+              rule[:source_groups] << source_group
+            end if ip_item["groups"]
+
+            ip_item["ipRanges"].each do |ip_range|
+              rule[:source_ips] ||= []
+              rule[:source_ips] << ip_range["cidrIp"]
+            end if ip_item["ipRanges"]
+
+            group[:permissions] << rule
+          end
+
+          groups << group
+        end
+
+        groups
+      end
+
+      def create_volume(size, zone)
+        volume = @compute_provider.volumes.create(:size => size.to_s, :availability_zone => zone)
+        return volume.id
+      end
+
+      def attach_volume(volume_id, instance_id, device)
+        volume = @compute_provider.volumes.get(volume_id)
+        server = @compute_provider.servers.get(instance_id)
+        volume.device = device
+        volume.server = server
+      end
+
+      def detach_volume(volume_id, force=true)
+        volume = @compute_provider.volumes.get(volume_id)
+        force ? volume.force_detach : (volume.server = nil)
+      end
+
+      def describe_volumes(volume_id=nil)
+        volumes = []
+        opts = {}
+        opts[:'volume-id'] = volume_id if volume_id
+        response = @compute_provider.volumes.all(opts)
+        response.each do |item|
+          volume = {}
+          volume[:id] = item.id
+          volume[:status] = item.state
+          if item.server_id
+            volume[:attachment_instance_id] = item.server_id
+            volume[:attachment_status] = item.attached_at ? "attached" : "waiting"
+          end
+          volumes << volume
+        end
+        return volumes
+      end
+
+      def destroy_volume(volume_id)
+        @compute_provider.volumes.get(volume_id).destroy
+      end
+
+      # resource_id is any Amazon resource ID (e.g., instance ID or volume ID)
+      # tags is a hash of tag_name => tag_value pairs
+      def create_tags(resource_id, tags)
+        # Tags need to be created individually in fog
+        tags.each do |k, v|
+          @compute_provider.tags.create(:resource_id => resource_id,
+                                        :key => k.to_s, :value => v.to_s)
+        end
+      end
+
+      private
+
+      def create_security_group(group_name, group_description)
+        @compute_provider.security_groups.create(:name => group_name, :description => group_description)
+      end
+
+      def destroy_security_group(group_name)
+        @compute_provider.security_groups.get(group_name).destroy
+      end
+
+      def add_security_group_rule(group_name, protocol, from_port, to_port, source)
+        group = @compute_provider.security_groups.get(group_name)
+        opts = {:ip_protocol => protocol || 'tcp'}
+
+        if source.instance_of? Hash
+          opts[:group] = {source[:account] => source[:name]}
+        else
+          opts[:cidr_ip] = source
+        end
+
+        group.authorize_port_range(from_port.to_i..to_port.to_i, opts)
+      end
+
+      def remove_security_group_rule(group_name, protocol, from_port, to_port, source)
+        group = @compute_provider.security_groups.get(group_name)
+        opts = {:ip_protocol => protocol || 'tcp'}
+
+        if source.instance_of? Hash
+          opts[:group] = {source[:account] => source[:name]}
+        else
+          opts[:cidr_ip] = source
+        end
+
+        group.revoke_port_range(from_port.to_i..to_port.to_i, opts)
+      end
+
+      def sync_security_groups(groups)
+        return unless groups
+
+        groups = Rubber::Util::stringify(groups)
+        groups = isolate_groups(groups)
+        group_keys = groups.keys.clone()
+
+        # For each group that does already exist in cloud
+        cloud_groups = describe_security_groups()
+        cloud_groups.each do |cloud_group|
+          group_name = cloud_group[:name]
+
+          # skip those groups that don't belong to this project/env
+          next if env.isolate_security_groups && group_name !~ /^#{isolate_prefix}/
+
+          if group_keys.delete(group_name)
+            # sync rules
+            capistrano.logger.debug "Security Group already in cloud, syncing rules: #{group_name}"
+            group = groups[group_name]
+
+            # convert the special case default rule into what it actually looks like when
+            # we query ec2 so that we can match things up when syncing
+            rules = group['rules'].clone
+            group['rules'].each do |rule|
+              if [2, 3].include?(rule.size) && rule['source_group_name'] && rule['source_group_account']
+                rules << rule.merge({'protocol' => 'tcp', 'from_port' => '1', 'to_port' => '65535' })
+                rules << rule.merge({'protocol' => 'udp', 'from_port' => '1', 'to_port' => '65535' })
+                rules << rule.merge({'protocol' => 'icmp', 'from_port' => '-1', 'to_port' => '-1' })
+                rules.delete(rule)
+              end
+            end
+
+            rule_maps = []
+
+            # first collect the rule maps from the request (group/user pairs are duplicated for tcp/udp/icmp,
+            # so we need to do this up frnot and remove duplicates before checking against the local rubber rules)
+            cloud_group[:permissions].each do |rule|
+              source_groups = rule.delete(:source_groups)
+              if source_groups
+                source_groups.each do |source_group|
+                  rule_map = rule.clone
+                  rule_map.delete(:source_ips)
+                  rule_map[:source_group_name] = source_group[:name]
+                  rule_map[:source_group_account] = source_group[:account]
+                  rule_map = Rubber::Util::stringify(rule_map)
+                  rule_maps << rule_map unless rule_maps.include?(rule_map)
+                end
+              else
+                rule_map = Rubber::Util::stringify(rule)
+                rule_maps << rule_map unless rule_maps.include?(rule_map)
+              end
+            end if cloud_group[:permissions]
+            # For each rule, if it exists, do nothing, otherwise remove it as its no longer defined locally
+            rule_maps.each do |rule_map|
+              if rules.delete(rule_map)
+                # rules match, don't need to do anything
+                # logger.debug "Rule in sync: #{rule_map.inspect}"
+              else
+                # rules don't match, remove them from cloud and re-add below
+                answer = nil
+                msg = "Rule '#{rule_map.inspect}' exists in cloud, but not locally"
+                if env.prompt_for_security_group_sync
+                  answer = Capistrano::CLI.ui.ask("#{msg}, remove from cloud? [y/N]: ")
+                else
+                  capistrano.logger.info(msg)
+                end
+
+                if answer =~ /^y/
+                  rule_map = Rubber::Util::symbolize_keys(rule_map)
+                  if rule_map[:source_group_name]
+                    remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
+                  else
+                    rule_map[:source_ips].each do |source_ip|
+                      remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
+                    end if rule_map[:source_ips]
+                  end
+                end
+              end
+            end
+
+            rules.each do |rule_map|
+              # create non-existing rules
+              capistrano.logger.debug "Missing rule, creating: #{rule_map.inspect}"
+              rule_map = Rubber::Util::symbolize_keys(rule_map)
+              if rule_map[:source_group_name]
+                add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
+              else
+                rule_map[:source_ips].each do |source_ip|
+                  add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
+                end if rule_map[:source_ips]
+              end
+            end
+          else
+            # delete group
+            answer = nil
+            msg = "Security group '#{group_name}' exists in cloud but not locally"
+            if env.prompt_for_security_group_sync
+              answer = Capistrano::CLI.ui.ask("#{msg}, remove from cloud? [y/N]: ")
+            else
+              capistrano.logger.debug(msg)
+            end
+            destroy_security_group(group_name) if answer =~ /^y/
+          end
+        end
+
+        # For each group that didnt already exist in cloud
+        group_keys.each do |group_name|
+          group = groups[group_name]
+          capistrano.logger.debug "Creating new security group: #{group_name}"
+          # create each group
+          create_security_group(group_name, group['description'])
+          # create rules for group
+          group['rules'].each do |rule_map|
+            capistrano.logger.debug "Creating new rule: #{rule_map.inspect}"
+            rule_map = Rubber::Util::symbolize_keys(rule_map)
+            if rule_map[:source_group_name]
+              add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
+            else
+              rule_map[:source_ips].each do |source_ip|
+                add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
+              end if rule_map[:source_ips]
+            end
+          end
+        end
+      end
     end
 
   end

data/lib/rubber/cloud/base.rb

@@ -10,6 +10,158 @@ module Rubber
         @capistrano = capistrano
       end
 
+      def before_create_instance(instance_alias, role_names)
+        # No-op by default.
+      end
+
+      def after_create_instance(instance)
+        # No-op by default.
+      end
+
+      def before_refresh_instance(instance)
+        # No-op by default.
+      end
+
+      def after_refresh_instance(instance)
+        setup_security_groups(instance.name, instance.role_names)
+      end
+
+      def isolate_prefix
+        "#{env.app_name}_#{Rubber.env}_"
+      end
+
+      def active_state
+        raise NotImplementedError, "active_state not implemented in base adapter"
+      end
+
+      def isolate_group_name(group_name)
+        if env.isolate_security_groups
+          group_name =~ /^#{isolate_prefix}/ ? group_name : "#{isolate_prefix}#{group_name}"
+        else
+          group_name
+        end
+      end
+
+      def isolate_groups(groups)
+        renamed = {}
+
+        groups.each do |name, group|
+          new_name = isolate_group_name(name)
+          new_group =  Marshal.load(Marshal.dump(group))
+
+          new_group['rules'].each do |rule|
+            old_ref_name = rule['source_group_name']
+            if old_ref_name
+              # don't mangle names if the user specifies this is an external group they are giving access to.
+              # remove the external_group key to allow this to match with groups retrieved from cloud
+              is_external = rule.delete('external_group')
+              if ! is_external && old_ref_name !~ /^#{isolate_prefix}/
+                rule['source_group_name'] = isolate_group_name(old_ref_name)
+              end
+            end
+          end
+
+          renamed[new_name] = new_group
+        end
+
+        renamed
+      end
+
+      def inject_auto_security_groups(groups, hosts, roles)
+        hosts.each do |name|
+          group_name = name
+          groups[group_name] ||= {'description' => "Rubber automatic security group for host: #{name}", 'rules' => []}
+        end
+        roles.each do |name|
+          group_name = name
+          groups[group_name] ||= {'description' => "Rubber automatic security group for role: #{name}", 'rules' => []}
+        end
+
+        groups
+      end
+
+      def setup_security_groups(host=nil, roles=[])
+        raise "Digital Ocean provider can only set up one host a time" if host.split(',').size != 1
+
+        rubber_cfg = Rubber::Configuration.get_configuration(Rubber.env)
+        scoped_env = rubber_cfg.environment.bind(roles, host)
+        security_group_defns = Hash[scoped_env.security_groups.to_a]
+
+
+        if scoped_env.auto_security_groups
+          sghosts = (scoped_env.rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
+          sgroles = (scoped_env.rubber_instances.all_roles + roles).uniq.compact
+          security_group_defns = inject_auto_security_groups(security_group_defns, sghosts, sgroles)
+        end
+
+        groups = Rubber::Util::stringify(security_group_defns)
+        groups = isolate_groups(groups)
+
+        script = <<-ENDSCRIPT
+          # Clear out all firewall rules to start.
+          iptables -F
+
+          iptables -I INPUT 1 -i lo -j ACCEPT -m comment --comment 'Enable connections on loopback devices.'
+          iptables -I INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT -m comment --comment 'Always allow established connections to remain connected.'
+        ENDSCRIPT
+
+        instance = scoped_env.rubber_instances[host]
+        instance.security_groups.each do |group_name|
+          group = groups[group_name]
+
+          group['rules'].each do |rule|
+            protocol = rule['protocol']
+            from_port = rule.has_key?('from_port') ? rule['from_port'].to_i : nil
+            to_port = rule.has_key?('to_port') ? rule['to_port'].to_i : nil
+            source_ips = rule['source_ips']
+
+            if protocol && from_port && to_port && source_ips
+              (from_port..to_port).each do |port|
+                source_ips.each do |source|
+                  script << "\niptables -A INPUT -p #{protocol} --dport #{port} --source #{source} -j ACCEPT -m comment --comment '#{group_name}'"
+                end
+              end
+            end
+          end
+        end
+
+        script << "\niptables -A INPUT -j DROP -m comment --comment 'Disable all other connections.'"
+
+        capistrano.run_script 'setup_firewall_rules', script, :hosts => instance.external_ip
+      end
+
+      def describe_security_groups(group_name=nil)
+        rules = capistrano.capture("iptables -S INPUT", :hosts => rubber_env.rubber_instances.collect(&:external_ip)).strip.split("\r\n")
+        scoped_rules = rules.select { |r| r =~ /dport/ }
+
+        groups = []
+
+        scoped_rules.each do |rule|
+          group = {}
+          discovered_rule = {}
+
+          parts = rule.split(' ').each_slice(2).to_a
+          parts.each do |arg, value|
+            case arg
+              when '-p' then discovered_rule[:protocol] = value
+              when '--dport' then discovered_rule[:from_port] = value; discovered_rule[:to_port] = value
+              when '--comment' then group[:name] = value
+            end
+          end
+
+          # Consolidate rules for groups with the same name.
+          existing_group = groups.find { |g| g[:name] == group[:name]}
+          if existing_group
+            existing_group[:permissions] << discovered_rule
+          else
+            group[:permissions] = [discovered_rule]
+            groups << group
+          end
+        end
+
+        groups
+      end
+
     end
 
   end