rubber 2.1.0 → 2.1.1

data/.travis.yml

@@ -3,9 +3,9 @@ rvm:
   - 1.8.7
   - 1.9.2
   - 1.9.3
-  - jruby-18mode
+#  - jruby-18mode
   - jruby-19mode
-  - jruby-head
+#  - jruby-head
 #  - rbx-18mode
 #  - rbx-19mode
 # script: bundle exec rspec spec

data/CHANGELOG

@@ -1,3 +1,36 @@
+2.1.1 (10/02/2012)
+------------------
+
+New Features:
+============
+[base] add ability to override by environment in rubber*.yml <f4c4af3>
+[couchbase, monit] add couchbase template <550dff4>
+
+Improvements:
+============
+[core] Updated sudo_script to sudo to any user. <288ef5d>
+[apache, jenkins] create an apache bootstrap so jenkins template doesn't have to duplicate logic <5862791>
+[complete_passenger, passenger] do a rolling passenger restart by tieing into apache rolling restart <065d757>
+[complete_passenger] dont do health check over ssl by default (Fails with self signed cert) <a5a8da7>
+[graylog] Updated graylog from 0.9.6 to 0.9.6p1. <864795c>
+[graylog] Use puma instead of passenger for graylog-web, since it works much better with JRuby. <7419705>
+[jenkins] Added support for backing up Jenkins. <a44162d>
+[jenkins] Fixed Jenkins config. <bbaa6f4>
+[jenkins] Make sure we have the header module enabled for the config to work. <93d03d8>
+[jenkins] The module is "headers", not "header". <4057334>
+[passenger] Bumped rubber from 3.0.12 to 3.0.15. <bc408f1>
+[passenger_nginx] Updated Nginx config to improve performance of asset pipeline. <7184327>
+[torquebox] Updated TorqueBox config from 2.0.3 to 2.1.0. <13a0268>
+
+Bug Fixes:
+=========
+[core] fix merging of alias records <d314857>
+[core] Refactor to avoid fog deprecation, fixes #222 <43ea42f>
+[core] assign defaults for partial records in rubber-dns.yml when running setup_dns_records <a72a701>
+[monit] monit check for resque scheduler - fixes #221 <34a4fde>
+[mysql] Delete anonymous MySQL user when bootstrapping the db. Fixes #185 (pull request #212) <ce84ff2>
+[torquebox] Always use the rubber system JRuby with Torquebox. <9c03a02>
+
 2.1.0 (08/20/2012)
 ------------------
 

data/lib/rubber/cloud/fog.rb

@@ -171,30 +171,28 @@ module Rubber
 
       def add_security_group_rule(group_name, protocol, from_port, to_port, source)
         group = @compute_provider.security_groups.get(group_name)
+        opts = {:ip_protocol => protocol || 'tcp'}
+        
         if source.instance_of? Hash
-          if protocol
-            groupstr = "#{source[:account]}:#{source[:name]}"
-            group.authorize_port_range(from_port.to_i..to_port.to_i, :ip_protocol => protocol, :group => groupstr)
-          else
-            group.authorize_group_and_owner(source[:name], source[:account])
-          end
+          opts[:group] = {source[:account] => source[:name]}
         else
-          group.authorize_port_range(from_port.to_i..to_port.to_i, :ip_protocol => protocol, :cidr_ip => source)
+          opts[:cidr_ip] = source
         end
+        
+        group.authorize_port_range(from_port.to_i..to_port.to_i, opts)
       end
 
       def remove_security_group_rule(group_name, protocol, from_port, to_port, source)
         group = @compute_provider.security_groups.get(group_name)
+        opts = {:ip_protocol => protocol || 'tcp'}
+        
         if source.instance_of? Hash
-          if protocol
-            groupstr = "#{source[:account]}:#{source[:name]}"
-            group.revoke_port_range(from_port.to_i..to_port.to_i, :ip_protocol => protocol, :group => groupstr)
-          else
-            group.revoke_group_and_owner(source[:name], source[:account])
-          end
+          opts[:group] = {source[:account] => source[:name]}
         else
-          group.revoke_port_range(from_port.to_i..to_port.to_i, :ip_protocol => protocol, :cidr_ip => source)
+          opts[:cidr_ip] = source
         end
+        
+        group.revoke_port_range(from_port.to_i..to_port.to_i, opts)
       end
 
       def destroy_security_group(group_name)

data/lib/rubber/configuration.rb

@@ -34,7 +34,7 @@ module Rubber
       def initialize(env=nil, root=nil)
         @env = env
         @root = root || "#{Rubber.root}/config/rubber"
-        @environment = Environment.new("#{@root}")
+        @environment = Environment.new("#{@root}", @env)
       end
 
       def load

data/lib/rubber/environment.rb

@@ -9,11 +9,14 @@ module Rubber
     # the host/role passed into bind
     class Environment
       attr_reader :config_root
+      attr_reader :config_env
       attr_reader :config_files
       attr_reader :config_secret
 
-      def initialize(config_root)
+      def initialize(config_root, env)
         @config_root = config_root
+        @config_env = env
+        
         @config_files = ["#{@config_root}/rubber.yml"]
         @config_files += Dir["#{@config_root}/rubber-*.yml"].sort
 
@@ -71,7 +74,7 @@ module Rubber
       end
       
       def bind(roles = nil, host = nil)
-        BoundEnv.new(@items, roles, host)
+        BoundEnv.new(@items, roles, host, config_env)
       end
 
       # combine old and new into a single value:
@@ -160,10 +163,12 @@ module Rubber
       class BoundEnv < HashValueProxy
         attr_reader :roles
         attr_reader :host
+        attr_reader :env
 
-        def initialize(global, roles, host)
+        def initialize(global, roles, host, env)
           @roles = roles
           @host = host
+          @env = env
           bound_global = bind_config(global)
           super(nil, bound_global)
         end
@@ -176,12 +181,16 @@ module Rubber
         def bind_config(global)
           global = global.clone()
           role_overrides = global.delete("roles") || {}
+          env_overrides = global.delete("environments") || {}
           host_overrides = global.delete("hosts") || {}
           Array(roles).each do |role|
             Array(role_overrides[role]).each do |k, v|
               global[k] = Environment.combine(global[k], v)
             end
           end
+          Array(env_overrides[env]).each do |k, v|
+            global[k] = Environment.combine(global[k], v)
+          end
           Array(host_overrides[host]).each do |k, v|
             global[k] = Environment.combine(global[k], v)
           end

data/lib/rubber/recipes/rubber/setup.rb

@@ -173,7 +173,12 @@ namespace :rubber do
       record[:type] ||= r[:type]
       record[:ttl] ||= r[:ttl] if r[:ttl]
       record[:data] ||= []
-      record[:data].concat(Array(r[:data]))
+      case r[:data]
+        when nil then ;
+        when Array then record[:data].concat(r[:data])
+        else
+          record[:data] << r[:data]
+      end
     end
     return record
   end
@@ -194,6 +199,7 @@ namespace :rubber do
       rubber_records = {}
       records.each do |record|
         record = Rubber::Util.symbolize_keys(record)
+        record = provider.setup_opts(record) # assign defaults        
         key = record_key(record)
         rubber_records[key] ||= []
         rubber_records[key] << record

data/lib/rubber/recipes/rubber/utils.rb

@@ -170,9 +170,12 @@ namespace :rubber do
   end
 
   def sudo_script(name, contents, opts = {})
+    user = opts.delete(:as)
     args = opts.delete(:script_args)
     script = prepare_script(name, contents)
-    run "#{sudo} bash -l #{script} #{args}", opts
+
+    sudo_args = user ? "-H -u #{user}" : ""
+    run "#{sudo} #{sudo_args} bash -l #{script} #{args}", opts
   end
 
   def top.rsudo(command, opts = {}, &block)

data/lib/rubber/version.rb

@@ -1,4 +1,3 @@
 module Rubber
-  VERSION = "2.1.0"
+  VERSION = "2.1.1"
 end
-

data/rubber.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'thor'
   s.add_dependency 'clamp'
   s.add_dependency 'open4'
-  s.add_dependency 'fog', '~> 1.5'
+  s.add_dependency 'fog', '~> 1.6'
   s.add_dependency 'json'
   
   s.add_development_dependency('rake')

data/templates/apache/config/rubber/deploy-apache.rb

@@ -18,14 +18,25 @@ namespace :rubber do
         fi
       ENDSCRIPT
     end
+    
+    after "rubber:bootstrap", "rubber:apache:bootstrap"
+
+    task :bootstrap, :roles => :apache do
+      exists = capture("grep 'empty ports file' /etc/apache2/ports.conf || true")
+      if exists.strip.size == 0
+        rubber.update_code_for_bootstrap
+        rubber.run_config(:file => "role/apache", :force => true, :deploy_path => release_path)
+      end
+    end
+    
 
     # serial_task can only be called after roles defined - not normally a problem, but
     # rubber auto-roles don't get defined till after all tasks are defined
     on :load do
-      rubber.serial_task self, :serial_restart, :roles => :apache do
+      rubber.serial_task self, :serial_restart, :roles => [:app, :apache] do
         rsudo "service apache2 stop; service apache2 start"
       end
-      rubber.serial_task self, :serial_reload, :roles => :apache do
+      rubber.serial_task self, :serial_reload, :roles => [:app, :apache] do
         rsudo "if ! ps ax | grep -v grep | grep -c apache2 &> /dev/null; then service apache2 start; else service apache2 reload; fi"
       end
     end

data/templates/base/config/rubber/rubber.yml

@@ -302,8 +302,8 @@ stop_on_error_cmd: "function error_exit { exit 99; }; trap error_exit ERR"
 #
 # var_name: var_value
 
-# All variables can also be overridden on the role and/or host level by creating
-# a sub level to the config under roles and hosts
+# All variables can also be overridden on the role, environment and/or host level by creating
+# a sub level to the config under roles, environments and hosts.  The precedence is host, environment, role
 # e.g. to install mysql only on db role, and awstats only on web01:
 
 # OPTIONAL: Role specific overrides
@@ -313,6 +313,13 @@ stop_on_error_cmd: "function error_exit { exit 99; }; trap error_exit ERR"
 #   somerole2:
 #     myconfig: someval
 
+# OPTIONAL: Environment specific overrides
+# environments:
+#   staging:
+#     myconfig: otherval
+#   production:
+#     myconfig: val
+
 # OPTIONAL: Host specific overrides
 # hosts:
 #   somehost:

data/templates/complete_passenger/config/rubber/role/haproxy/haproxy-passenger.conf

@@ -8,14 +8,21 @@
 %>
 
 listen passenger_proxy 0.0.0.0:<%= rubber_env.web_port %>
+  monitor-uri /haproxy_test
   option forwardfor
+  <% if Rubber.env == 'production' %>
+    option httpchk OPTIONS /httpchk.txt
+  <% end %>
   <% backend_hosts.each do |server| %>
     server <%= server %> <%= server %>:<%= rubber_env.passenger_listen_port %> maxconn <%= rubber_env.max_app_connections %> check
   <% end %>
 
 listen passenger_proxy_ssl 0.0.0.0:<%= rubber_env.web_ssl_port %>
   mode tcp
-  option forwardfor
+  <% if Rubber.env == 'production' %>
+    # only uncomment if you have a valid ssl cert
+    # option httpchk OPTIONS /httpchk.txt
+  <% end %>
   <% backend_hosts.each do |server| %>
     server <%= server %> <%= server %>:<%= rubber_env.passenger_listen_ssl_port %> maxconn <%= rubber_env.max_app_connections %> check
   <% end %>

data/templates/couchbase/config/rubber/deploy-couchbase.rb

@@ -0,0 +1,105 @@
+namespace :rubber do
+
+  namespace :couchbase do
+
+    rubber.allow_optional_tasks(self)
+
+    before "rubber:install_packages", "rubber:couchbase:setup_apt_sources"
+  
+    # sources needed for couchbase ruby client, not server
+    task :setup_apt_sources do
+      sources = <<-SOURCES
+        # Ubuntu 11.10 Oneiric Ocelot (Debian unstable)
+        #
+        # preview version needed for ruby client that has rails cache/session
+        #
+        # deb http://packages.couchbase.com/ubuntu oneiric oneiric/main
+        deb http://packages.couchbase.com/preview/ubuntu oneiric oneiric/main
+      SOURCES
+      sources.gsub!(/^[ \t]*/, '')
+      put(sources, "/etc/apt/sources.list.d/couchbase.list") 
+      rsudo "wget -O- http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add -"
+    end
+
+    after "rubber:install_packages", "rubber:couchbase:install"
+
+    task :install, :roles => :couchbase do
+      rubber.sudo_script 'install_couchbase', <<-ENDSCRIPT
+        if ! grep "^#{rubber_env.couchbase_version}" /opt/couchbase/VERSION.txt &> /dev/null; then
+          # Fetch and install the pkg
+          rm -rf /tmp/couchbase*
+          wget -qNP /tmp #{rubber_env.couchbase_pkg_url}
+          dpkg -i /tmp/couchbase*
+
+          # Clean up after ourselves.
+          rm -rf /tmp/couchbase*
+        fi
+      ENDSCRIPT
+    end
+
+    after "rubber:bootstrap", "rubber:couchbase:bootstrap"
+
+    task :bootstrap, :roles => :couchbase do
+      exists = capture("echo $(ls #{rubber_env.couchbase_db_dir} 2> /dev/null)")
+      if exists.strip.size == 0
+        
+        cli = rubber_env.couchbase_cli
+        create_bucket_lines = []
+        ram_size = 0
+        rubber_env.couchbase_buckets.each do |bucket_spec|
+          create_bucket_lines << "#{cli} bucket-create $cluster --bucket=#{bucket_spec['name']} --bucket-type=#{bucket_spec['type']} --bucket-port=#{bucket_spec['port']} --bucket-ramsize=#{bucket_spec['size']} --bucket-replica=#{bucket_spec['replicas']}"
+          ram_size += bucket_spec['size'].to_i
+        end
+        
+        rubber.sudo_script 'bootstrap_couchbase', <<-ENDSCRIPT
+          mkdir -p #{rubber_env.couchbase_db_dir}
+          chown -R couchbase:couchbase #{rubber_env.couchbase_db_dir}
+          local_ip=$(ifconfig eth0 | awk -F"[: ]+" 'NR==2 {print $4}')
+          cluster="-c $local_ip:8091 -u #{rubber_env.couchbase_admin_username} -p #{rubber_env.couchbase_admin_password}"
+
+          # Setup cluster
+          #{cli} cluster-init $cluster --cluster-init-ramsize=#{ram_size}
+
+          # initialize the node
+          #{cli} node-init $cluster --node-init-data-path=#{rubber_env.couchbase_db_dir}
+
+          # create the buckets
+          #{create_bucket_lines.join("\n")}
+
+        ENDSCRIPT
+  
+        # After everything installed on machines, we need the source tree
+        # on hosts in order to run rubber:config for bootstrapping the db
+        rubber.update_code_for_bootstrap
+  
+        # Gen just the conf for couchbase
+        rubber.run_config(:file => "role/couchbase/", :force => true, :deploy_path => release_path)
+      
+        restart
+      end
+    end
+
+    desc "Stops the couchbase server"
+    task :stop, :roles => :couchbase, :on_error => :continue do
+      rsudo "service couchbase-server stop || true"
+    end
+
+    desc "Starts the couchbase server"
+    task :start, :roles => :couchbase do
+      rsudo "service couchbase-server start"
+    end
+
+    desc "Restarts the couchbase server"
+    task :restart, :roles => :couchbase do
+      stop
+      start
+    end
+
+    desc "Reloads the couchbase server"
+    task :reload, :roles => :couchbase do
+      restart
+    end
+
+  end
+
+end

data/templates/couchbase/config/rubber/role/couchbase/crontab

@@ -0,0 +1,8 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ['# start-couchbase-crontab', '# end-couchbase-crontab']
+%>
+
+# Backup couchbase DBs every 3 hours.
+# 0 */3 * * * <%= Rubber.root %>/script/rubber cron --task util:backup --directory=/mnt/couchbase_backups --name=couchbase --command="nice cbbackup  <%= rubber_env.couchbase_db_dir %> \%dir\%/\%name\%_<%= rubber_env.host %>_\%time_stamp\%"

data/templates/couchbase/config/rubber/rubber-couchbase.yml

@@ -0,0 +1,25 @@
+couchbase_version: 1.8.1
+couchbase_pkg_url: "http://packages.couchbase.com/releases/#{couchbase_version}/couchbase-server-community_x86_64_#{couchbase_version}.deb"
+
+couchbase_pid_file: /opt/couchbase/var/lib/couchbase/couchbase-server.pid
+couchbase_cli: /opt/couchbase/bin/couchbase-cli
+couchbase_db_dir: /mnt/couchbase/data
+
+couchbase_admin_username: Administrator
+couchbase_admin_password: password
+
+couchbase_buckets:
+  - name: "default"
+    type: couchbase
+    size: 200
+    replicas: 1
+    port: 11211
+  - name: "#{app_name}"
+    type: couchbase
+    size: 200
+    replicas: 1
+    port: 11222
+
+gems: [couchbase]
+
+packages: [libcouchbase-dev]

data/templates/couchbase/templates.rb

@@ -0,0 +1,3 @@
+if Rubber::Util::is_bundler?
+  append_to_file "Gemfile", "gem 'couchbase'\n"
+end

data/templates/couchbase/templates.yml

@@ -0,0 +1 @@
+description: The couchbase module

data/templates/graylog/config/rubber/deploy-graylog.rb

@@ -81,9 +81,13 @@ namespace :rubber do
           rubber.update_code_for_bootstrap
 
           rubber.run_config(:file => "role/graylog_web/", :force => true, :deploy_path => release_path)
-          
+
           rubber.sudo_script 'bootstrap_graylog_web', <<-ENDSCRIPT
             cd #{rubber_env.graylog_web_dir}
+
+            # Add puma to the Gemfile so we can run the server.
+            echo "gem 'puma'" >> Gemfile
+
             export RAILS_ENV=production
             bundle install