rubber 2.0.5 → 2.0.6

data/lib/rubber/dns/zerigo.rb

@@ -1,16 +1,122 @@
 require 'rubygems'
 require 'fog'
-require 'rubber/dns/fog'
 
 module Rubber
   module Dns
 
-    class Zerigo < Fog
+    class Zerigo < Base
 
+      attr_accessor :client
+          
       def initialize(env)
-        super(env.merge({"credentials" => { "provider" => 'zerigo', "zerigo_email" => env.email, "zerigo_token" => env.token }}))
+        super(env)
+        creds = { :provider => 'zerigo', :zerigo_email => env.email, :zerigo_token => env.token }
+        @client = ::Fog::DNS.new(creds)
       end
-    
+      
+      # multiple hosts with same name/type convert to a single rubber-dns.yml opts format
+      def hosts_to_opts(hosts)
+        opts = {}
+        
+        hosts.each do |host|
+          opts[:host] ||= host.name || ''
+          opts[:domain] ||= host.zone.domain
+          opts[:type] ||= host.type
+          opts[:ttl] ||= host.ttl.to_i if host.ttl
+          
+          opts[:data] ||= []
+          if host.type =~ /MX/i
+            opts[:data] << {:priority => host.priority, :value => host.value}
+          else
+            opts[:data] << host.value
+          end
+        end
+        
+        return opts
+      end
+
+      # a single rubber-dns.yml opts format converts to multiple hosts with same name/type 
+      def opts_to_hosts(opts)
+        hosts = []
+        
+        opts[:data].each do |o|
+          host = {}
+          host[:name] = opts[:host]
+          host[:type] =  opts[:type]
+          host[:ttl] = opts[:ttl] if opts[:ttl]
+          if o.kind_of?(Hash) && o[:priority]
+            host[:priority] = o[:priority]
+            host[:value] = o[:value]
+          else
+            host[:value] = o
+          end
+          hosts << host
+        end
+        
+        return hosts
+      end
+
+      def find_or_create_zone(domain)
+        zone = @client.zones.all.find {|z| z.domain =~ /^#{domain}\.?/}
+        if ! zone
+          zone = @client.zones.create(:domain => domain)
+        end
+        return zone
+      end
+      
+      def find_hosts(opts = {})
+        opts = setup_opts(opts, [:host, :domain])
+        result = []
+        zone = find_or_create_zone(opts[:domain])
+
+        # TODO: revert this when zerigo fog gets fixed to allow parameters 
+        # hosts = fqdn ? (zone.records.all(:name => fqdn) rescue []) : zone.records.all
+        hosts = zone.records.all
+        hosts = hosts.select {|h| name = h.name || ''; name == opts[:host] } if opts.has_key?(:host) && opts[:host] != '*'
+        hosts = hosts.select {|h| h.type == opts[:type] } if opts.has_key?(:type) && opts[:type] != '*'
+        
+        return hosts
+      end
+
+      def find_host_records(opts = {})
+        hosts = find_hosts(opts)
+        group = {}
+        hosts.each do |h|
+          key = "#{h.name}.#{h.domain} #{h.type}"
+          group[key] ||= []
+          group[key] << h
+        end
+        result = group.values.collect {|h| hosts_to_opts(h).merge(:domain => opts[:domain])}
+        return result
+      end
+
+      def create_host_record(opts = {})
+        opts = setup_opts(opts, [:host, :data, :domain, :type, :ttl])
+        zone = find_or_create_zone(opts[:domain])
+        opts_to_hosts(opts).each do |host|
+          zone.records.create(host)
+        end
+      end
+
+      def destroy_host_record(opts = {})
+        opts = setup_opts(opts, [:host, :domain])
+
+        find_hosts(opts).each do |h|
+          h.destroy || raise("Failed to destroy #{h.hostname}")
+        end
+      end
+
+      def update_host_record(old_opts={}, new_opts={})
+        old_opts = setup_opts(old_opts, [:host, :domain, :type])
+        new_opts = setup_opts(new_opts, [:host, :domain, :type, :data])
+
+        # Tricky to update existing hosts since zerigo needs a separate host
+        # entry for multiple records of same type (MX, etc), so take the easy
+        # way out and destroy/create instead of update
+        destroy_host_record(old_opts)
+        create_host_record(new_opts)
+      end
+
     end
     
   end

data/lib/rubber/instance.rb

@@ -41,7 +41,7 @@ module Rubber
             bucket = location.split("/")[0]
             key = location.split("/")[1..-1].join("/")
             data = Rubber.cloud.storage(bucket).fetch(key)
-            StringIO.open(data, 'r') {|f| load_from_file(f) }
+            StringIO.open(data, 'r') {|f| load_from_file(f) } if data
           when /table:(.*)/
             location = $1
             load_from_table(location)

data/lib/rubber/recipes/rubber/instances.rb

@@ -68,9 +68,11 @@ namespace :rubber do
     Reboot the EC2 instance for the give ALIAS
   DESC
   required_task :reboot do
-    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)
+    
+    aliases = Rubber::Util::parse_aliases(instance_aliases)
     ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
-    reboot_instance(instance_alias)
+    reboot_instances(aliases, ENV['FORCE'] =~ /^(t|y)/)
   end
 
   desc <<-DESC
@@ -443,15 +445,21 @@ namespace :rubber do
     setup_aliases
   end
   
+  def reboot_instances(instance_aliases, force=false)
+    instance_aliases.each do |instance_alias|
+      reboot_instance(instance_alias, force)
+    end
+  end
+  
   # Reboots the given ec2 instance
-  def reboot_instance(instance_alias)
+  def reboot_instance(instance_alias, force=false)
     instance_item = rubber_instances[instance_alias]
     fatal "Instance does not exist: #{instance_alias}" if ! instance_item
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    value = Capistrano::CLI.ui.ask("About to REBOOT #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
-    fatal("Exiting", 0) if value != "yes"
+    value = Capistrano::CLI.ui.ask("About to REBOOT #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ") unless force
+    fatal("Exiting", 0) if value != "yes" && ! force
 
     logger.info "Rebooting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
 

data/lib/rubber/recipes/rubber/security_groups.rb

@@ -43,7 +43,7 @@ namespace :rubber do
 
   def setup_security_groups(host=nil, roles=[])
     env = rubber_cfg.environment.bind(roles, host)
-    security_group_defns = env.security_groups
+    security_group_defns = Hash[env.security_groups.to_a]
     if env.auto_security_groups
       sghosts = (rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
       sgroles = (rubber_instances.all_roles + roles).uniq.compact

data/lib/rubber/recipes/rubber/setup.rb

@@ -161,80 +161,96 @@ namespace :rubber do
     end
   end
 
+  def record_key(record)
+    "#{record[:host]}.#{record[:domain]} #{record[:type]}"
+  end
+  
+  def convert_to_new_dns_format(records)
+    record = {}
+    records.each do |r|
+      record[:host] ||= r[:host]
+      record[:domain] ||= r[:domain]
+      record[:type] ||= r[:type]
+      record[:ttl] ||= r[:ttl] if r[:ttl]
+      record[:data] ||= []
+      record[:data].concat(Array(r[:data]))
+    end
+    return record
+  end
+
   desc <<-DESC
     Sets up the additional dns records supplied in the dns_records config in rubber.yml
   DESC
   required_task :setup_dns_records do
     records = rubber_env.dns_records
     if records && rubber_env.dns_provider
-      provider = Rubber::Dns::get_provider(rubber_env.dns_provider, rubber_env)
-
-      # collect the round robin records (those with the same host/domain/type)
-      rr_records = []
-      records.each_with_index do |record, i|
-        m = records.find_all {|r| record['host'] == r['host'] && record['domain'] == r['domain'] && record['type'] == r['type']}
-        m = m.sort {|a,b| a.object_id <=> b.object_id}
-        rr_records << m if m.size > 1 && ! rr_records.include?(m)
-      end
-
-      # simple records are those that aren't round robin ones
-      simple_records = records - rr_records.flatten
       
-      # for each simple record, create or update as necessary
-      simple_records.each do |record|
-        matching = provider.find_host_records(:host => record['host'], :domain =>record['domain'], :type => record['type'])
-        if matching.size > 1
-          msg =  "Multiple records in dns provider, but not in rubber.yml\n"
-          msg << "Round robin records need to be in both, or neither.\n"
-          msg << "Please fix manually:\n"
-          msg << matching.pretty_inspect
-          fatal(msg)
-        end
-
-        record = provider.setup_opts(record)
-        if matching.size == 1
-          match = matching.first
-          if provider.host_records_equal?(record, match)
-            logger.info "Simple dns record already up to date: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
-          else
-            logger.info "Updating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
-            provider.update_host_record(match, record)
-          end
-        else
-          logger.info "Creating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
-          provider.create_host_record(record)
-        end
+      provider_name = rubber_env.dns_provider
+      provider = Rubber::Dns::get_provider(provider_name, rubber_env)
+      
+      # records in rubber_env.dns_records can either have a value which
+      # is an array, or multiple equivalent (same host+type)items with
+      # value being a string, so try and normalize them
+      rubber_records = {}
+      records.each do |record|
+        record = Rubber::Util.symbolize_keys(record)
+        key = record_key(record)
+        rubber_records[key] ||= []
+        rubber_records[key] << record
+      end
+      rubber_records = Hash[rubber_records.collect {|key, records| [key, convert_to_new_dns_format(records)] }]
+
+      provider_records = {}
+      domains = rubber_records.values.collect {|r| r[:domain] }.uniq
+      precords = domains.collect {|d| provider.find_host_records(:host => '*', :type => '*', :domain => d) }.flatten
+      precords.each do |record|
+        key = record_key(record)
+        raise "unmerged provider records" if provider_records[key] 
+        provider_records[key] = record
       end
 
-      # group round robin records
-      rr_records.each do |rr_group|
-        host = rr_group.first['host']
-        domain = rr_group.first['domain']
-        type = rr_group.first['type']
-        matching = provider.find_host_records(:host => host, :domain => domain, :type => type)
-
-        # remove from consideration the local records that are the same as remote ones
-        matching.clone.each do |r|
-          rr_group.delete_if {|rg| provider.host_records_equal?(r, rg) }
-          matching.delete_if {|rg| provider.host_records_equal?(r, rg) }
-        end
-        if rr_group.size == 0 && matching.size == 0
-          logger.info "Round robin dns records already up to date: #{host}.#{domain}:#{type}"
-        end
-
-        # create the local records that don't exist remotely
-        rr_group.each do |r|
-          r = provider.setup_opts(r)
-          logger.info "Creating round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
-          provider.create_host_record(r)
-        end
-        
-        # remove the remote records that don't exist locally
-        matching.each do |r|
-          logger.info "Removing round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
-          provider.destroy_host_record(r)
+      changes = Hash[(rubber_records.to_a - provider_records.to_a) | (provider_records.to_a - rubber_records.to_a)]
+
+      logger.info "Applying #{changes.size} changes"
+      changes.each do |key, record|
+        old_record = provider_records[key]
+        new_record = rubber_records[key]
+        if old_record && new_record
+          # already exists in provider, so modify it
+          diff = Hash[(old_record.to_a - new_record.to_a) | (new_record.to_a - old_record.to_a)]
+          logger.info "Updating dns record: #{old_record.inspect} changes: #{diff.inspect}"
+          #provider.update_host_record(old_record, new_record)
+        elsif !old_record && new_record
+          # doesn't yet exist in provider, so create it
+          logger.info "Creating dns record: #{new_record.inspect}"
+          #provider.create_host_record(new_record)
+        elsif old_record && ! new_record
+          # ignore these since it shows all the instances created by rubber
+          #
+          #logger.info "Provider record doesn't exist locally: #{old_record.inspect}"
+          #if ENV['FORCE']
+          #  destroy_dns_record = ENV['FORCE'] =~ /^(t|y)/
+          #else
+          #  destroy_dns_record = get_env('DESTROY_DNS', "Destroy DNS record in provider [y/N]?", true)
+          #end
+          #provider.destroy_host_record(old_record) if destroy_dns_record
         end
       end
+
+    end
+  end
+  
+  desc <<-DESC
+    Exports dns records from your provider into the format readable by rubber in rubber-dns.yml
+  DESC
+  required_task :export_dns_records do
+    if rubber_env.dns_provider
+      
+      provider_name = rubber_env.dns_provider
+      provider = Rubber::Dns::get_provider(provider_name, rubber_env)
+      
+      provider_records = provider.find_host_records(:host => '*', :type => '*', :domain => rubber_env.domain)
+      puts({'dns_records' => provider_records.collect {|r| Rubber::Util.stringify_keys(r)}}.to_yaml)
     end
   end
 
@@ -371,7 +387,7 @@ namespace :rubber do
   task :set_timezone do
     opts = get_host_options('timezone')
     rsudo "echo $CAPISTRANO:VAR$ > /etc/timezone", opts
-    rsudo "cp /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
+    rsudo "ln -sf /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
     # restart syslog so that times match timezone
     sudo_script 'restart_syslog', <<-ENDSCRIPT
       if [[ -x /etc/init.d/sysklogd ]]; then

data/lib/rubber/util.rb

@@ -10,6 +10,13 @@ module Rubber
       end
     end
     
+    def stringify_keys(map)
+      map.inject({}) do |options, (key, value)|
+        options[key.to_s || key] = value
+        options
+      end
+    end
+    
     def stringify(val)
       case val
       when String

data/lib/rubber/version.rb

@@ -1,4 +1,4 @@
 module Rubber
-  VERSION = "2.0.5"
+  VERSION = "2.0.6"
 end
 

data/templates/base/config/rubber/deploy-setup.rb

@@ -103,5 +103,17 @@ namespace :rubber do
       ENDSCRIPT
     end
 
+    # Update /etc/sudoers so that SSH-related environment variables so capistrano/rubber tasks can take advantage of ssh-agent forwarding
+    before "rubber:bootstrap", "rubber:base:update_sudoers"
+    task :update_sudoers do
+      rubber.sudo_script "update_sudoers", <<-ENDSCRIPT
+        if [[ ! `grep 'SSH_CLIENT SSH_TTY SSH_CONNECTION SSH_AUTH_SOCK' /etc/sudoers` =~ "SSH_CLIENT SSH_TTY SSH_CONNECTION SSH_AUTH_SOCK" ]]; then
+          echo '' >> /etc/sudoers
+          echo '# whitelist SSH-related environment variables so capistrano tasks can take advantage of ssh-agent forwarding' >> /etc/sudoers
+          echo 'Defaults env_keep += "SSH_CLIENT SSH_TTY SSH_CONNECTION SSH_AUTH_SOCK"' >> /etc/sudoers
+        fi
+      ENDSCRIPT
+    end
+
   end
 end

data/templates/base/config/rubber/rubber-dns.yml

@@ -5,11 +5,9 @@
 #          This lets rubber update a dynamic dns service with the instance alias and ip
 #
 dns_providers:
-  fog:
-    credentials:
-     provider: aws
-     aws_access_key_id: "#{cloud_providers.aws.access_key}"
-     aws_secret_access_key: "#{cloud_providers.aws.secret_access_key}"
+  aws:
+    access_key: "#{cloud_providers.aws.access_key}"
+    access_secret: "#{cloud_providers.aws.secret_access_key}"
     type: A
     ttl: 300
   nettica:
@@ -35,18 +33,18 @@ dns_providers:
 #  dns_records:
 #    # simple A record
 #    - host: bar
-#      data: 1.1.1.1
+#      data: [1.1.1.1]
 #
 #    # more detailed A record
 #    - host: bar
 #      domain: otherdomain.com
-#      data: 1.1.1.1
+#      data: [1.1.1.1]
 #      type: A
 #      ttl: 300
 #
 #    # tld A record
 #    - host: ''
-#      data: 1.1.1.1
+#      data: [1.1.1.1]
 #      type: A
 #
 #    # tld ALIAS record -- Route53 extension to point tld at an Elastic Load Balancer group
@@ -54,40 +52,38 @@ dns_providers:
 #      domain: example.com
 #      type: A
 #      data:
-#        hosted_zone_id: XXYYZZ
-#        dns_name: my-app-elb-group.us-east-1.elb.amazonaws.com
+#        - hosted_zone_id: XXYYZZ
+#          dns_name: my-app-elb-group.us-east-1.elb.amazonaws.com
 #
 #    # simple CNAME record
 #    - host: otherbar
 #      domain: foo.com
-#      data: bar.foo.com
+#      data: [bar.foo.com]
 #      type: CNAME
 #      ttl: 300
 #
-#    # 2 of the same A records is a round robin dns
+#    # Round robin dns record
 #    - host: rr
 #      domain: foo.com
-#      data: 1.1.1.1
-#      type: A
-#      ttl: 300
-#    - host: rr
-#      domain: foo.com
-#      data: 1.1.1.2
+#      data: [1.1.1.1, 1.1.1.2]
 #      type: A
 #      ttl: 300
 #
 #    # A record, grabbing ip from instance config
 #    - host: baz
 #      domain: foo.com
-#      data: "#{rubber_instances.for_role('web').first.external_ip}"
+#      data: ["#{rubber_instances.for_role('web').first.external_ip}"]
 #      type: A
 #      ttl: 300
 #
 #    # MX record
 #    - host: ''
 #      domain: foo.com
-#      data: mail.foo.com
+#      data:
+#        - priority: 5
+#          value: mail1.foo.com
+#        - priority: 10
+#          value: mail2.foo.com
 #      type: MX
 #      ttl: 300
-#      priority: 10