rubber 2.0.5 → 2.0.6

data/.travis.yml

@@ -3,8 +3,8 @@ rvm:
   - 1.8.7
   - 1.9.2
   - 1.9.3
-  - jruby-18mode
-  - jruby-19mode
+#  - jruby-18mode
+#  - jruby-19mode
 #  - jruby-head
 #  - rbx-18mode
 #  - rbx-19mode

data/CHANGELOG

@@ -1,3 +1,53 @@
+2.0.6 (08/01/2012)
+------------------
+
+New Features:
+============
+[resque_scheduler] Merge pull request #194 from messick/resque_scheduler_template <dc20ecb>
+[resque_scheduler] Rake task file must end in .rake not .rb <03a7cfa>
+[resque_scheduler] append to log on restart <eadd00a>
+[resque_scheduler] enable scheduler tabs in resque-web ui <3e68509>
+[resque_scheduler] first pass at a resque scheduler template <e9e0111>
+[resque_scheduler] log output from resque-scheduler upstart job <91c3790>
+[resque_scheduler] redirect error to log as well <d75b96a>
+[resque_scheduler] use resque_scheduler PID path from yaml file. <e699193>
+
+Improvements:
+============
+[core] multi-host rubber:reboot <56ef06f>
+[core] update nettica to work with new ruber-dns.yml format <1bfa525>
+[base] format change to better support multi-value records like MX, round robin A, etc refactor to better handle route53/zerigo differences in fog add export_dns_records to allow dumping from provider to a rubber-dns.yml format <7c806cf>
+[base] update base ami <be8c0f4>
+
+Bug Fixes:
+=========
+[core] isoalte security groups from global namespace so having a key with the same name as a capistrano role doesn't cause problems <41ba2cf>
+[core] make cron logs more user friendly and prevent exponential explosion of files in log dir due to interaction with rubber:util:rotate_logs <7e11a7f>
+[core] fix nettica support (fix super call, credentials) <14fd048>
+[core] Merge pull request #192 from garrytan/master <837d8be>
+[core] don't attempt to read a non-existant instance file in cloud storage <c4d11ac>
+[core] Merge pull request #173 from viniciusnz/master <d7e8f9f>
+[core] Avoid error if localtime file already exists <cff1fb4>
+[base] Merge pull request #190 from davebenvenuti/ssh_env_whitelist <48260c9>
+[base] whitelist ssh-related variables in sudoers so capistrano tasks can take advantage of ssh-agent forwarding <1009e7b>
+[collectd] Merge pull request #191 from davebenvenuti/passenger_sudoers_fix <09511a8>
+[collectd] add to env variable whitelist rather than overwrite it <dff23f7>
+[elasticsearch] Ensure unzip package present for elasticsearch <40af636>
+[graphite] fix for multiple targets in graph defn <b72c469>
+[jenkins] Merge pull request #171 from ScotterC/patch-1 <133aa3e>
+[jenkins] removed hardcoded application, thanks to scotterc for finding this <b5b26b4>
+[jenkins] sun-java6 fails on Ubuntu 10.04 <88779a9>
+[passenger] Removed the code to remove config.ru from Rails apps as it's no longer needed in Passenger 3. <517d23e>
+[postgresql] Fixed an issue with PostgreSQL messing with casing on us. <72e63b5>
+[resque] Merge pull request #174 from messick/resque_fix <dc70092>
+[resque] Merge pull request #179 from messick/resque_redis_fix <d2d436c>
+[resque] connect to redis_master role, rather than just redis <f778dc8>
+[resque] explicitly make Rubber.root a string <c0852b2>
+[resque] placeholder config for dev <d855381>
+[resque] rails_env was changed to Rubber.env a while ago I believe <2376db2>
+[torquebox] Ensure unzip package present for torquebox <7f4d7aa>
+[zookeeper] fix zookeeper server ids <3273b53>
+
 2.0.5 (06/12/2012)
 ------------------
 

data/lib/rubber/commands/cron.rb

@@ -56,28 +56,25 @@ module Rubber
       parameter "COMMAND ...", "the command to run"
 
       def execute
-        self.rootdir ||= Rubber.root
-        self.logfile ||= "#{Rubber.root}/log/cron-sh-#{Time.now.tv_sec}.log"
-        
         cmd = command_list
+        self.rootdir ||= Rubber.root
+        ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
+        self.logfile ||= "#{Rubber.root}/log/cron-sh-#{ident}.log"
         log = logfile
         
         if task?
-          ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
-          log = "#{rootdir}/log/cron-task-#{ident}-#{Time.now.tv_sec}.log"
+          log = "#{rootdir}/log/cron-task-#{ident}.log"
           cmd = ["rubber"] + cmd
         elsif ruby?
           ruby_code = cmd.join(' ')
           ident = ruby_code.gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
-          log = "#{rootdir}/log/cron-ruby-#{ident}-#{Time.now.tv_sec}.log"
+          log = "#{rootdir}/log/cron-ruby-#{ident}.log"
           cmd = ["ruby", "-e", ruby_code]
         elsif runner?
-          ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
-          log = "#{rootdir}/log/cron-runner-#{ident}-#{Time.now.tv_sec}.log"
+          log = "#{rootdir}/log/cron-runner-#{ident}.log"
           cmd = ["rails", "runner"] + cmd
         elsif rake?
-          ident = cmd[0].gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]
-          log = "#{rootdir}/log/cron-rake-#{ident}-#{Time.now.tv_sec}.log"
+          log = "#{rootdir}/log/cron-rake-#{ident}.log"
           cmd = ["rake"] + cmd
         end
         
@@ -99,7 +96,8 @@ module Rubber
         # Open4 forks, which JRuby doesn't support.  But JRuby added a popen4-compatible method on the IO class,
         # so we can use that instead.
         status = (defined?(JRUBY_VERSION) ? IO : Open4).popen4(*cmd) do |pid, stdin, stdout, stderr|
-          File.open(log, "w") do | fh |
+          File.open(log, "a") do | fh |
+            fh.puts "\nrubber:cron running #{cmd.inspect} at #{Time.now}\n"
             threads = []
             threads <<  Thread.new(stdout) do |stdout|
                stdout.each { |line| $stdout.puts line if echoout?; fh.print line; fh.flush }

data/lib/rubber/commands/util.rb

@@ -1,3 +1,4 @@
+require 'date'
 
 module Rubber
   module Commands

data/lib/rubber/dns.rb

@@ -4,10 +4,35 @@ module Rubber
   module Dns
 
     def self.get_provider(provider, env)
-      require "rubber/dns/#{provider}"
-      clazz = Rubber::Dns.const_get(provider.capitalize)
-      provider_env = env.dns_providers[provider]
-      return clazz.new(provider_env)
+
+      if provider == 'fog'
+        # TODO: remove backwards compatibility in next major release
+        
+        puts "deprecated dns provider config: #{env.inspect}"
+        provider_env = env.dns_providers['fog']
+        creds = provider_env.credentials
+        real_provider = creds.provider
+        require "rubber/dns/#{real_provider}"
+        clazz = Rubber::Dns.const_get(real_provider.capitalize)
+        case real_provider
+          when 'aws'
+            provider_env['access_key'] = creds['aws_access_key_id']
+            provider_env['access_secret'] = creds['aws_secret_access_key']
+          when 'zerigo'
+            provider_env['email'] = creds['zerigo_email']
+            provider_env['token'] = creds['zerigo_token']
+        end
+        return clazz.new(provider_env)
+        
+      else
+        
+        require "rubber/dns/#{provider}"
+        clazz = Rubber::Dns.const_get(provider.capitalize)
+        provider_env = env.dns_providers[provider]
+        return clazz.new(provider_env)
+        
+      end
+      
     end
     
   end

data/lib/rubber/dns/aws.rb

@@ -0,0 +1,181 @@
+require 'rubygems'
+require 'fog'
+
+module Rubber
+  module Dns
+
+    class Aws < Base
+
+      attr_accessor :client
+
+      def initialize(env)
+        super(env)
+        creds = { :provider => 'aws', :aws_access_key_id => env.access_key, :aws_secret_access_key => env.access_secret }
+        @client = ::Fog::DNS.new(creds)
+      end
+      
+      def normalize_name(name, domain)
+        domain = domain.gsub(/\.$/, "")
+
+        if name
+          name = name.gsub(/\.$/, "")
+          name = name.gsub(/.?#{domain}$/, "")
+          # Route 53 encodes asterisks in their ASCII octal representation.
+          name = name.gsub("\\052", "*")
+        end
+        
+        return name, domain
+      end
+      
+      def denormalize_name(name, domain)
+        if ! name || name.strip.empty?
+          name = "#{domain}"
+        else
+          name = "#{name}.#{domain}"
+        end
+
+        name = "#{name}." 
+        domain = "#{domain}."
+        
+        return name, domain
+      end
+      
+      # Convert from fog/aws model to rubber option hash that represents a dns record
+      def host_to_opts(host)
+        opts = {}
+        
+        opts[:host] ||= host.name || ''
+        opts[:domain] ||= host.zone.domain
+        opts[:host], opts[:domain] = normalize_name(opts[:host], opts[:domain])
+        
+        opts[:type] ||= host.type
+        opts[:ttl] ||= host.ttl.to_i if host.ttl
+        
+        opts[:data] ||= []
+        if host.type =~ /MX/i
+          host.value.each do |val|
+            parts = val.split(" ")
+            opts[:data] << {'priority' => parts[0], 'value' => parts[1]}
+          end
+        elsif ! host.alias_target.nil?
+          # Convert from camel-case to snake-case for Route 53 ALIAS records
+          # so the match the rubber config format.
+          opts[:data] << {
+            'hosted_zone_id' => host.alias_target['HostedZoneId'],
+            'dns_name' => host.alias_target['DNSName'].split('.')[0..-1].join('.')
+          }
+          # Route 53 ALIAS records do not have a TTL, so delete the rubber-supplied default value.
+          opts.delete(:ttl)
+        else
+          opts[:data].concat(Array(host.value))
+        end
+
+        return opts
+      end
+
+      # Convert from rubber option hash that represents a dns record to fog/aws model 
+      def opts_to_host(opts)
+        host = {}
+        host[:name], domain = denormalize_name(opts[:host], opts[:domain])
+        
+        host[:type] =  opts[:type]
+        host[:ttl] = opts[:ttl] if opts[:ttl]
+
+        if opts[:data]
+          # Route 53 requires the priority to be munged with the data value.
+          if host[:type] =~ /MX/i
+            host[:value] = opts[:data].collect {|o| "#{o[:priority]} #{o[:value]}"}
+          elsif opts[:data].first.is_a?(Hash)
+            # Route 53 allows creation of ALIAS records, which will always be
+            # a Hash in the DNS config.  ALIAS records cannot have a TTL.
+            host[:alias_target] = opts[:data].first
+            host.delete(:value)
+            host.delete(:ttl)
+          else
+            host[:value] = opts[:data]
+          end
+        end
+
+        return host
+      end
+
+      def find_or_create_zone(domain)
+        zone = @client.zones.all.find {|z| z.domain =~ /^#{domain}\.?/}
+        if ! zone
+          zone = @client.zones.create(:domain => domain)
+        end
+        return zone
+      end
+      
+      def all_hosts(zone)
+        hosts = []
+        opts = {}
+        has_more = true
+        
+        while has_more
+          all_hosts = zone.records.all(opts)
+          has_more = all_hosts.is_truncated
+          opts = {:name => all_hosts.next_record_name,
+                  :type => all_hosts.next_record_type
+          }
+          hosts.concat(all_hosts)
+        end
+        
+        return hosts        
+      end
+      
+      
+      def find_hosts(opts = {})
+        opts = setup_opts(opts, [:host, :domain])
+        result = []
+        zone = find_or_create_zone(opts[:domain])
+        host = opts_to_host(opts)
+        
+        if opts[:host] && opts[:host] != '*'
+          found_host = zone.records.all(:name => host[:name], :type => host[:type], :max_items => 1).first
+          found_host = nil if found_host.name != "#{host[:name]}." && found_host.type != host[:type]
+          hosts = Array(found_host)
+        else
+          hosts = all_hosts(zone)
+        end
+        
+        hosts = hosts.select {|h| h.name == host[:name] } if opts.has_key?(:host) && opts[:host] != '*'
+        hosts = hosts.select {|h| h.type == host[:type] } if opts.has_key?(:type) && opts[:type] != '*'
+        
+        return hosts
+      end
+
+      def find_host_records(opts = {})
+        hosts = find_hosts(opts)
+        result = hosts.collect {|h| host_to_opts(h).merge(:domain => opts[:domain]) }
+        return result
+      end
+
+      def create_host_record(opts = {})
+        opts = setup_opts(opts, [:host, :data, :domain, :type, :ttl])
+        zone = find_or_create_zone(opts[:domain])
+        zone.records.create(opts_to_host(opts))
+      end
+
+      def destroy_host_record(opts = {})
+        opts = setup_opts(opts, [:host, :domain])
+
+        find_hosts(opts).each do |h|
+          h.destroy || raise("Failed to destroy #{h.hostname}")
+        end
+      end
+
+      def update_host_record(old_opts={}, new_opts={})
+        old_opts = setup_opts(old_opts, [:host, :domain, :type])
+        new_opts = setup_opts(new_opts, [:host, :domain, :type, :data])
+        new_host = opts_to_host(new_opts)
+
+        host = find_hosts(old_opts).first
+        result = host.modify(new_host)
+        result || raise("Failed to update host #{host.name}, #{host.errors.full_messages.join(', ')}")
+      end
+      
+    end
+
+  end
+end

data/lib/rubber/dns/nettica.rb

@@ -10,8 +10,8 @@ module Rubber
     class Nettica < Base
 
       def initialize(env)
-        super(env, "nettica")
-        @client = ::Nettica::Client.new(provider_env.user, provider_env.password)
+        super(env)
+        @client = ::Nettica::Client.new(env.user, env.password)
       end
 
       def check_status(response)
@@ -31,13 +31,12 @@ module Rubber
         return response
       end
 
-      def find_host_records(opts = {})
+      def find_hosts(opts = {})
         opts = setup_opts(opts, [:host, :domain])
 
         result = []
         hn = opts[:host]
         ht = opts[:type]
-        hd = opts[:data]
 
         domain_info = find_or_create_zone(opts[:domain])
 
@@ -49,37 +48,49 @@ module Rubber
           if ht && h.recordType != ht && ht != '*'
             keep = false
           end
-          if hd && h.data != hd
-            keep = false
-          end
-          result << record_to_opts(h) if keep
+          result << h if keep
         end
 
         return result
       end
 
+      def find_host_records(opts = {})
+        hosts = find_hosts(opts)
+        group = {}
+        hosts.each do |h|
+          key = "#{h.hostName}.#{h.domainName} #{h.recordType}"
+          group[key] ||= []
+          group[key] << h
+        end
+        result = group.values.collect {|h| hosts_to_opts(h).merge(:domain => opts[:domain])}
+        return result
+      end
+      
       def create_host_record(opts = {})
         opts = setup_opts(opts, [:host, :data, :domain, :type, :ttl])
         find_or_create_zone(opts[:domain])
-        record = opts_to_record(opts)
-        check_status @client.add_record(record)
+        opts_to_hosts(opts).each do |host|
+          check_status @client.add_record(host)
+        end
       end
 
       def destroy_host_record(opts = {})
-        find_host_records(opts).each do |h|
-          record = opts_to_record(h)
-          check_status @client.delete_record(record)
+        opts = setup_opts(opts, [:host, :domain])
+
+        find_hosts(opts).each do |h|
+          check_status @client.delete_record(h)
         end
       end
 
       def update_host_record(old_opts = {}, new_opts = {})
-        old_opts = setup_opts(old_opts, [:host, :domain])
-        new_opts = setup_opts(new_opts.merge(:no_defaults =>true), [])
-        find_host_records(old_opts).each do |h|
-          old_record = opts_to_record(h)
-          new_record = opts_to_record(h.merge(new_opts))
-          check_status @client.update_record(old_record, new_record)
-        end
+        old_opts = setup_opts(old_opts, [:host, :domain, :type])
+        new_opts = setup_opts(new_opts, [:host, :domain, :type, :data])
+
+        # Tricky to update existing hosts since nettica needs a separate host
+        # entry for multiple records of same type (MX, etc), so take the easy
+        # way out and destroy/create instead of update
+        destroy_host_record(old_opts)
+        create_host_record(new_opts)
       end
 
       private
@@ -95,27 +106,54 @@ module Rubber
         end
         return domain_info
       end
+      
+      # multiple hosts with same name/type convert to a single rubber-dns.yml opts format
+      def hosts_to_opts(hosts)
+        opts = {}
+        
+        hosts.each do |record|
+          opts[:host] ||= record.hostName || ''
+          opts[:domain] ||= record.domainName
+          opts[:type] ||= record.recordType
+          opts[:ttl] ||= record.tTL if record.tTL
+
+          opts[:data] ||= []
+          if opts[:type] =~ /MX/i
+            opts[:data] << {:priority => record.priority, :value => record.data}
+          else
+            opts[:data] << record.data
+          end
+        end
+        
+        return opts
+      end
 
-      def opts_to_record(opts)
-        record = @client.create_domain_record(opts[:domain],
+      # a single rubber-dns.yml opts format converts to multiple hosts with same name/type 
+      def opts_to_hosts(opts)
+        hosts = []
+        
+        opts[:data].each do |o|
+          
+          data, priority = nil, 0
+          if o.kind_of?(Hash) && o[:priority]
+            priority = o[:priority]
+            data = o[:value]
+          else
+            data = o
+          end
+          
+          host = @client.create_domain_record(opts[:domain],
                                               opts[:host],
                                               opts[:type],
-                                              opts[:data],
+                                              data,
                                               opts[:ttl],
-                                              opts[:priority] || 0)
-        return record
-      end
-
-      def record_to_opts(record)
-        opts = {}
-        opts[:host] = record.hostName || ''
-        opts[:domain] = record.domainName
-        opts[:type] = record.recordType
-        opts[:data] = record.data if record.data
-        opts[:ttl] = record.tTL if record.tTL
-        opts[:priority] = record.priority if record.priority
-        return opts
+                                              priority)
+          hosts << host
+        end
+        
+        return hosts
       end
+      
     end
 
   end