rubber 2.5.4 → 2.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5eee7ca6b10464295c9a962aef561ca84f03579d
-  data.tar.gz: 7e592fc7c63e42e11a5d05aa7e031fbca679a46b
+  metadata.gz: ec1e12978f936ceec04c66adb0faaee47bd40b58
+  data.tar.gz: f9d9ad4325c684e4d51c7ac677e96e4613c70107
 SHA512:
-  metadata.gz: 29bb8877ceb4e3542829197561418cd1a07e8d5872ffb94a0a2459bb64c7caea67584dd4cf41d6da380381a250f80da213a38636235f2d0477a70f7f280568b6
-  data.tar.gz: fe9a45483916c0c9812a5c28c325d0c119cbb351e135f07dcf5404faa9e4a54e8adc4a857df59dc4847db76ee97151deb9489907aa2a242c91acb8ba4563f565
+  metadata.gz: e820367d0dce0f9636d08ffd9b2e94bf418cf575bd8b510754924f60b5d7e2a52471118b5e500c1a6ecfa38df9b752e6cfbd1cfd82684929d5c4d92cac8825ce
+  data.tar.gz: 546e2c34cb91151d0012b7c379d8964348960a9403344b7ffa536c0d620ba84bc5c06316299f7f92a46540768bc631c460e1e43f684e1f88c87ff8fe9cfbd29b

data/.travis.yml

@@ -41,6 +41,8 @@ matrix:
 
   allow_failures:
     - rvm: ruby-head
+    - rvm: jruby-head
+    - rvm: rbx-18mode
 
 # script: bundle exec rspec spec
 env:

data/CHANGELOG

@@ -1,3 +1,27 @@
+2.5.5 (10/16/2013)
+
+New Features:
+============
+
+[base, core, postgresql] Added a uniform way of handling private network address blocks <61c5e0c>
+
+Improvements:
+============
+
+[base] Vagrant base config now sets up key file for rubber to SSH into the machine <40c6c3c>
+[base] Default staging_roles config now reflects the cluster more accurately <255a8e9>
+[core] Only modify /etc/hosts on rubber:refresh if an IP address has actually changed <fe5e28e>
+[core] Support iptables rules without port definitions <de744c1>
+
+Bug Fixes:
+=========
+
+[core] Fixed a problem with tasks using the `:only` option being overwritten by rubber internally (#397) <d3ae2fa>
+[core] Don't install fog 1.16.0 until we're able to sort out API compatiblities with it <346c99b>
+[core] Presen the $REBOOT value across multiple calls to `maybe_reboot` <f03dc22>
+[web_tools] Proxied service hostnames should not have underscores since that would be an invalid hostname <055919f>
+
+
 2.5.4 (09/26/2013)
 
 Improvements:
@@ -17,6 +41,7 @@ Bug Fixes:
 [core] Wait up until 2 minutes for EC2 tag creation <3b6819a>
 [core] Set AWS region options for S3 <3bd20cd>
 [passenger_nginx] Removed conflicting passenger_min_instances configuration <cd7b006>
+[passenger_nginx] Added missing nginx_log_dir config <4684b26>
 [redis] Fixed redis download link <8a2a47d>
 
 

data/lib/rubber/cloud/base.rb

@@ -121,6 +121,12 @@ module Rubber
           iptables -I INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT -m comment --comment 'Always allow established connections to remain connected.'
         ENDSCRIPT
 
+        (scoped_env.private_networks || []).each do |network|
+          script << "\niptables -A INPUT -p tcp --dport 1:65535 --source #{network} -j ACCEPT -m comment --comment 'private_network_#{network}'"
+          script << "\niptables -A INPUT -p udp --dport 1:65535 --source #{network} -j ACCEPT -m comment --comment 'private_network_#{network}'"
+          script << "\niptables -A INPUT -p icmp -j ACCEPT -m comment --comment 'private_network_#{network}'"
+        end
+
         instance = scoped_env.rubber_instances[host]
         instance.security_groups.each do |group_name|
           group = groups[group_name]
@@ -131,12 +137,16 @@ module Rubber
             to_port = rule.has_key?('to_port') ? rule['to_port'].to_i : nil
             source_ips = rule['source_ips']
 
-            if protocol && from_port && to_port && source_ips
+            if protocol && source_ips
               source_ips.each do |source|
-                if from_port != to_port
-                  script << "\niptables -A INPUT -p #{protocol} --dport #{from_port}:#{to_port} --source #{source} -j ACCEPT -m comment --comment '#{group_name}'"
+                if from_port && to_port
+                  if from_port != to_port
+                    script << "\niptables -A INPUT -p #{protocol} --dport #{from_port}:#{to_port} --source #{source} -j ACCEPT -m comment --comment '#{group_name}'"
+                  else
+                    script << "\niptables -A INPUT -p #{protocol} --dport #{to_port} --source #{source} -j ACCEPT -m comment --comment '#{group_name}'"
+                  end
                 else
-                  script << "\niptables -A INPUT -p #{protocol} --dport #{to_port} --source #{source} -j ACCEPT -m comment --comment '#{group_name}'"
+                  script << "\niptables -A INPUT -p #{protocol} --source #{source} -j ACCEPT -m comment --comment '#{group_name}'"
                 end
               end
             end

data/lib/rubber/cloud/vagrant.rb

@@ -46,7 +46,10 @@ module Rubber
       end
 
       def destroy_instance(instance_id)
-        system("vagrant destroy #{instance_id} --force")
+        # If it's being run from vagrant, then 'vagrant destroy' must have been called already, so no need for us to do it.
+        unless ENV.has_key?('RUN_FROM_VAGRANT')
+          system("vagrant destroy #{instance_id} --force")
+        end
       end
 
       def stop_instance(instance, force=false)

data/lib/rubber/environment.rb

@@ -135,6 +135,10 @@ module Rubber
         def rubber_instances
           @rubber_instances ||= Rubber::Configuration::rubber_instances
         end
+        
+        def known_roles
+          Rubber::Configuration.get_configuration(Rubber.env).environment.known_roles
+        end
 
         def [](name)
           value = super(name)

data/lib/rubber/recipes/rubber.rb

@@ -13,7 +13,13 @@ namespace :rubber do
   # Disable connecting to any Windows instance.
   alias :original_task :task
   def task(name, options={}, &block)
-    original_task(name, options.merge(:only => { :platform => 'linux' }), &block)
+    if options.has_key?(:only)
+      options[:only][:platform] = 'linux'
+    else
+      options[:only] = { :platform => 'linux' }
+    end
+
+    original_task(name, options, &block)
   end
 
   # advise capistrano's task method so that tasks for non-existent roles don't
@@ -25,8 +31,13 @@ namespace :rubber do
     class << ns
       alias :required_task :task
       def task(name, options={}, &block)
-        # Disable connecting to any Windows instance.
-        required_task(name, options.merge(:only => { :platform => 'linux' })) do
+        if options.has_key?(:only)
+          options[:only][:platform] = 'linux'
+        else
+          options[:only] = { :platform => 'linux' }
+        end
+
+        required_task(name, options) do
           # define empty roles for the case when a task has a role that we don't define anywhere
           unless options[:roles].respond_to?(:call)
             [*options[:roles]].each do |r|
@@ -57,7 +68,12 @@ namespace :rubber do
     # use a pty so we don't get "stdin: is not a tty" error output
     default_run_options[:pty] = true if default_run_options[:pty].nil?
     default_run_options[:shell] = "/bin/bash -l" if default_run_options[:shell].nil?
-    default_run_options[:only] ||= { :platform => 'linux' }
+
+    if default_run_options.has_key?(:only)
+      default_run_options[:only][:platform] = 'linux'
+    else
+      default_run_options[:only] = { :platform => 'linux' }
+    end
 
     set :cloud, Rubber.cloud(self)
 

data/lib/rubber/recipes/rubber/setup.rb

@@ -124,11 +124,16 @@ namespace :rubber do
     local_hosts << delim << "\n"
 
     # Write out the hosts file for this machine, use sudo
-    filtered = File.read(hosts_file).gsub(/^#{delim}.*^#{delim}\n?/m, '')
-    logger.info "Writing out aliases into local machines #{hosts_file}, sudo access needed"
-    Rubber::Util::sudo_open(hosts_file, 'w') do |f|
-      f.write(filtered)
-      f.write(local_hosts)
+    existing = File.read(hosts_file)
+    filtered = existing.gsub(/^#{delim}.*^#{delim}\n?/m, '')
+
+    # only write out if it has changed
+    if existing != (filtered + local_hosts)
+      logger.info "Writing out aliases into local machines #{hosts_file}, sudo access needed"
+      Rubber::Util::sudo_open(hosts_file, 'w') do |f|
+        f.write(filtered)
+        f.write(local_hosts)
+      end
     end
   end
 
@@ -474,6 +479,7 @@ namespace :rubber do
       # graphite web app)
       if instance_item.role_names.include?('web_tools')
         Array(rubber_env.web_tools_proxies).each do |name, settings|
+          name = name.gsub('_', '-')
           provider.update("#{name}-#{instance_item.name}", instance_item.external_ip)
         end
       end
@@ -514,7 +520,11 @@ namespace :rubber do
 
     rsudo "apt-get -q update"
     if upgrade
-      rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes dist-upgrade"
+      if ENV['NO_DIST_UPGRADE']
+        rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes upgrade"
+      else
+        rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes dist-upgrade"
+      end
     else
       rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes install $CAPISTRANO:VAR$", opts
     end
@@ -556,6 +566,8 @@ namespace :rubber do
       end
 
       reboot = get_env('REBOOT', "Updates require a reboot on hosts #{reboot_hosts.inspect}, reboot [y/N]?", false)
+      ENV['REBOOT'] = reboot # `get_env` chomps the REBOOT value of the env, so reset it here so the value is retained across multiple calls.
+
       reboot = (reboot =~ /^y/)
 
       if reboot

data/lib/rubber/recipes/rubber/utils.rb

@@ -20,8 +20,6 @@ namespace :rubber do
       logger.info "Instance already exists, skipping to bootstrap"
     else
       default_roles = rubber_env.staging_roles
-      # default staging roles to all roles minus slaves (db without primary=true is a slave)
-      default_roles ||= rubber_cfg.environment.known_roles.reject {|r| r =~ /slave/ || r =~ /^db$/ }.join(",")
       roles = ENV['ROLES'] = rubber.get_env("ROLES", "Roles to use for staging instance", true, default_roles)
       
       rubber.create
@@ -108,7 +106,7 @@ namespace :rubber do
         next if servers[rolename].nil?
 
         servers[rolename] -= added_servers
-        added_servers << servers[rolename]
+        added_servers.concat(servers[rolename])
         servers[rolename] = servers[rolename].uniq.sort
       end
     end

data/lib/rubber/vagrant/provisioner.rb

@@ -23,6 +23,10 @@ module VagrantPlugins
         bootstrap && deploy_migrations
       end
 
+      def cleanup
+        destroy
+      end
+
       private
 
       def create
@@ -39,6 +43,20 @@ module VagrantPlugins
         system(script)
       end
 
+      def destroy
+        if config.use_vagrant_ruby
+          script = "RUN_FROM_VAGRANT=true FORCE=true RUBBER_ENV=#{config.rubber_env} ALIAS=#{machine.name} #{internal_cap_command} rubber:destroy"
+        else
+          script = <<-ENDSCRIPT
+            unset GEM_HOME;
+            unset GEM_PATH;
+            PATH=#{ENV['PATH'].split(':')[1..-1].join(':')} RUN_FROM_VAGRANT=true FORCE=true RUBBER_ENV=#{config.rubber_env} ALIAS=#{machine.name} bash -c '#{rvm_prefix} bundle exec cap rubber:destroy'
+          ENDSCRIPT
+        end
+
+        system(script)
+      end
+
       def refresh
         if config.use_vagrant_ruby
           script = "RUN_FROM_VAGRANT=true RUBBER_ENV=#{config.rubber_env} RUBBER_SSH_KEY=#{ssh_info[:private_key_path]} ALIAS=#{machine.name} EXTERNAL_IP=#{private_ip} INTERNAL_IP=#{private_ip} #{internal_cap_command} rubber:refresh -S initial_ssh_user=#{ssh_info[:username]}"

data/lib/rubber/version.rb

@@ -1,3 +1,3 @@
 module Rubber
-  VERSION = "2.5.4"
+  VERSION = "2.5.5"
 end

data/rubber.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'thor'
   s.add_dependency 'clamp'
   s.add_dependency 'open4'
-  s.add_dependency 'fog', '~> 1.6'
+  s.add_dependency 'fog', '~> 1.6', '< 1.16.0'
   s.add_dependency 'json'
   
   s.add_development_dependency('rake')

data/templates/apache/config/rubber/role/web_tools/tools-apache-vhost.conf

@@ -74,7 +74,7 @@ NameVirtualHost *:<%= rubber_env.web_tools_ssl_port %>
     proxy_host = rubber_instances.for_role(settings.role).first.full_name rescue nil
     next unless proxy_host
     
-    host = "#{name}-#{rubber_env.full_host}"
+    host = "#{name.gsub('_', '-')}-#{rubber_env.full_host}"
     host_and_port = "#{host}:#{rubber_env.web_tools_ssl_port}"
 
     # don't use settings.path here - mapping the host/port is sufficient,

data/templates/apache/config/rubber/role/web_tools/tools-index.html

@@ -30,7 +30,7 @@
         # graphite web app)
         Array(rubber_env.web_tools_proxies).each do |name, settings|
       %>
-        <li><a href="https://<%= name %>-<%= tools_host.full_name %>:<%= rubber_env.web_tools_ssl_port %><%= settings.path || '/' %>"><%= name.capitalize %></a></li>
+        <li><a href="https://<%= name.gsub('_', '-') %>-<%= tools_host.full_name %>:<%= rubber_env.web_tools_ssl_port %><%= settings.path || '/' %>"><%= name.capitalize %></a></li>
       <% end %>
 
 		</ul>

data/templates/base/config/rubber/rubber-vagrant-env.yml

@@ -1,7 +1,7 @@
 cloud_provider: vagrant
-
+ 
 cloud_providers:
   vagrant:
-    # Rubber assumes every cloud provider has configuration.  Vagrant really doesn't need anything, but until the core
-    # of Rubber handles that, just set a dummy value.
-    dummy: true
+    # Allow rubber to connect directly to instance in addition to use through vagrant commands.
+    key_name: insecure_private_key
+    key_file: "#{File.expand_path('~/.vagrant.d/insecure_private_key')}"

data/templates/base/config/rubber/rubber.yml

@@ -235,6 +235,11 @@ isolate_security_groups: true
 # differ from those in rubber
 prompt_for_security_group_sync: true
 
+# OPTIONAL: A list of CIDR address blocks that represent private networks for your cluster.
+# Set this to open up wide access to hosts in your network.  Consequently, setting the CIDR block
+# to anything other than a private, unroutable block would be a massive security hole.
+private_networks: [10.0.0.0/8]
+
 # OPTIONAL: The packages to install on all instances
 # You can install a specific version of a package by using a sub-array of pkg, version
 # For example, packages: [[rake, 0.7.1], irb]
@@ -260,7 +265,8 @@ stop_on_error_cmd: "function error_exit { exit 99; }; trap error_exit ERR"
 # specify a different set here
 #
 # staging_roles: "web,app,db:primary=true"
-
+# Auto detect staging roles
+staging_roles: "#{rubber_env.known_roles.reject {|r| r =~ /slave/ || r =~ /^db$/ }.join(',')}"
 
 # OPTIONAL: Lets one assign amazon elastic IPs (static IPs) to your instances
 #           You should typically set this on the role/host level rather than

data/templates/postgresql/config/rubber/role/postgresql/pg_hba.conf

@@ -95,6 +95,9 @@ local   all         <%= rubber_env.db_user %>                 <%= scheme %>
 host    all         all                         127.0.0.1/32  <%= scheme %>
 host    all         all                         ::1/128       <%= scheme %>
 
-host    replication <%= rubber_env.db_replication_user %>   10.0.0.1/8    <%= scheme %>
-host    all         <%= rubber_env.db_user %>               10.0.0.1/8    <%= scheme %>
+<% (rubber_env.private_networks || []).each do |network| %>
+host    replication <%= rubber_env.db_replication_user %>   <%= network %>    <%= scheme %>
+host    all         <%= rubber_env.db_user %>               <%= network %>    <%= scheme %>
+<% end %>
+
 host    all         <%= rubber_env.db_user %>               <%= rubber_instances[rubber_env.host].internal_ip %>/32    <%= scheme %>

data/test/test-rails-template.rb

@@ -53,3 +53,23 @@ run "cp -f #{secret} config/rubber/rubber-secret.yml"
 chmod 'config/rubber/rubber-secret.yml', 0644
 gsub_file 'config/rubber/rubber-secret.yml', /dns_provider: .*/, ''
 
+run "vagrant init precise32 http://files.vagrantup.com/precise32.box"
+vagrantfile = <<-EOS
+  config.vm.define :vagrant do |vagrant|
+    vagrant.vm.network :private_network, ip: "192.168.70.10"
+    
+    vagrant.vm.provider :virtualbox do |vb|
+      vb.customize ["modifyvm", :id, "--memory", "2048"]
+    end
+
+    vagrant.vm.provision :rubber do |rubber|
+      rubber.rubber_env = 'vagrant'
+
+      # Only necessary if you use RVM locally.
+      rubber.rvm_ruby_version = 'default'
+    end
+  end
+end
+EOS
+
+gsub_file 'Vagrantfile', /^end/, vagrantfile

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubber
 version: !ruby/object:Gem::Version
-  version: 2.5.4
+  version: 2.5.5
 platform: ruby
 authors:
 - Matt Conway
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-27 00:00:00.000000000 Z
+date: 2013-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capistrano
@@ -88,6 +88,9 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.6'
+    - - <
+      - !ruby/object:Gem::Version
+        version: 1.16.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -95,6 +98,9 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.6'
+    - - <
+      - !ruby/object:Gem::Version
+        version: 1.16.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -687,7 +693,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: rubber
-rubygems_version: 2.0.3
+rubygems_version: 2.1.7
 signing_key: 
 specification_version: 4
 summary: A capistrano plugin for managing multi-instance deployments to the cloud