rubber 2.6.5 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0994ced4a1d19f61e742eab894f3f268f398e50e
-  data.tar.gz: 5ad9bdec1b4ff61e4aa4984b0b75dc8940635274
+  metadata.gz: 8aebd49d6b6f4dc7f30e6ae01e61314f3237e407
+  data.tar.gz: 647342efba82bb0a7e224c90334a22fd19d73336
 SHA512:
-  metadata.gz: e54d4a074a8b624b8f7708d984d6f5636cb73dc5f03f60b1e2fc6cef9129aa6a10338d3761d506daf324a5bdd8c6018da19443f589d7775e84d78cfa5aee5a99
-  data.tar.gz: 448fcb8cc6a4a8c3e59fd8ea13dfcf2824bd2d51c796c531c5c876a4ae4bd552e1ab694b73dcce1e03132ac5884185d2712232ccd3c6709663cc1e49428afab9
+  metadata.gz: a75e1ff0966eef3fd8589b75056b4a78d11e86435a105e72aa2ee4cc2a24258f21e7da5f4d68ff49909d0b9a7985b1cd0b8da3fd568b90c74fe1e8eccb6c7004
+  data.tar.gz: 1e309a5b087198178f0aa8e28657947d078f727c205791d00b3e2efc4beab543017703ee502a1174e9b80af10b88defdb7e1790aae26de220a876d4ad95b4987

data/.travis.yml

@@ -41,6 +41,8 @@ matrix:
 
   allow_failures:
     - rvm: rbx-2
+    - rvm: 1.8.7
+    - rvm: jruby-18mode
 
 before_install:
   - gem install bundler -v '= 1.5.1'

data/CHANGELOG

@@ -1,3 +1,40 @@
+2.7.0 (02/03/2014)
+
+New Features:
+============
+
+[core] Added a new @delayed_post option to template preambles to defer executing a command until all templates have been processed. <41cae48>
+[delay_job] Added a new Delayed::Job role. <c7e33f5>
+[discourse] Added a new Discourse role that will take care of deploying the Discourse forum software and dependencies. <f01eeac>
+[ffmpeg] Added a new ffmpeg role. <23ba011>
+[newrelic] Added a new role for setting up the NewRelic system daemons. <7bc89ce>
+
+Improvements:
+============
+
+[core] Force shut down vSphere VMs when destroying an instance so the process isn't blocked if the machine doesn't shut down gracefully. <12bf770>
+[elasticsearch] Upgraded ElasticSearch from 0.19.12 to 0.90.10 and switched to the official deb packages. <a399316, 804bd1f, 4ca29e2>
+[graylog] Upgraded from graylog 0.9.6p1 to 0.20.0-rc1.1 <3476003, 4ca29e2>
+[memcached] Added a monit template. <87763e9>
+[memcached] Allow specifying the location of the PID file. <31bc9fb>
+[memcached] Major overhaul of the memcached template to modernize it. <8757cf8, 4307270, 8cebb56, ae8d89a>
+[mongodb] Monit will restart mongodb if the PID file doesn't exist. <7a57c3e>
+[redis] Upgraded from redis 2.6.16 to 2.8.4 <3877205, 644e6ee>
+[solr] Major overhaul of the solr template to modernize it. <a45046d, 2fc4779, 02e04b0, 676256d>
+
+Bug Fixes:
+=========
+
+[core] Fixed issue with calling cap rubber:setup_security_groups on multiple hosts. <bb17621>
+[core] Fixed volume creation on non-EC2 providers. <c49df54>
+[elasticsearch] Remove the elasticsearch-head plugin before attempting to reinstall it. <6438b68>
+[mysql] Delete all blank users initially to prevent issues with using a host value of '%'. <e75974a>
+[passenger_nginx] Added missing 'nginx_log_dir' setting to rubber-passenger_nginx.yml. <e6c3cdf>
+[passenger_nginx] Rebuild Passenger/Nginx when Ruby version changes. <fbb1e54>
+[solr] Fixed some typos in the Solr configuration. <a45046d>
+[unicorn] Fixed unicorn process management script. <c56a93c>
+
+
 2.6.5 (01/07/2014)
 
 Bug Fixes:

data/lib/rubber/cloud/vsphere.rb

@@ -98,6 +98,10 @@ module Rubber
         vm['new_vm']['id']
       end
 
+      def destroy_instance(instance_id)
+        compute_provider.servers.get(instance_id).destroy(:force => true)
+      end
+
       def describe_instances(instance_id=nil)
         instances = []
         opts = {}

data/lib/rubber/generator.rb

@@ -21,6 +21,7 @@ module Rubber
         @roles = roles.to_a.reverse #First roles take precedence
         @host = host || 'no_host'
         @options=options
+        @delayed_post_commands = []
       end
 
       def run
@@ -47,6 +48,33 @@ module Rubber
             end
           end
         end
+
+        @delayed_post_commands = @delayed_post_commands.uniq
+        if @delayed_post_commands.size > 0
+          Rubber.logger.info("Running delayed post commands")
+
+          @delayed_post_commands.each do |delayed_post_command|
+            run_post_command(delayed_post_command)
+          end
+        end
+      end
+
+      def run_post_command(cmd)
+        if fake_root
+          Rubber.logger.info("Not running post command as a fake root was given: #{cmd}")
+        elsif no_post
+          Rubber.logger.info("Not running post command as no post specified: #{cmd}")
+        else
+          # this lets us abort a script if a command in the middle of it errors out
+          # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
+          cmd_with_error_handler = stop_on_error_cmd ? "#{stop_on_error_cmd}\n#{cmd}" : cmd
+
+          Rubber.logger.info{"Transformation executing post config command: #{cmd}"}
+          Rubber.logger.info `#{cmd_with_error_handler}`
+          if $?.exitstatus != 0
+            raise "Post command failed execution:  #{cmd_with_error_handler}"
+          end
+        end
       end
 
       # Transforms the ERB template given in srcfile and writes the result to
@@ -154,25 +182,13 @@ module Rubber
           
           # Run post transform command if needed
           if config.post
-            if fake_root
-              Rubber.logger.info("Not running post command as a fake root was given: #{config.post}")
-            elsif no_post
-              Rubber.logger.info("Not running post command as no post specified")
-            else
-              if orig != result || force
-                # this lets us abort a script if a command in the middle of it errors out
-                # stop_on_error_cmd = "function error_exit { exit 99; }; trap error_exit ERR"
-                config.post = "#{stop_on_error_cmd}\n#{config.post}" if stop_on_error_cmd
-
-                Rubber.logger.info{"Transformation executing post config command: #{config.post}"}
-                Rubber.logger.info `#{config.post}`
-                if $?.exitstatus != 0
-                  raise "Post command failed execution:  #{config.post}"
-                end
-              else
-                Rubber.logger.info("Nothing to do, not running post command")
-              end
-            end
+            run_post_command(config.post)
+          end
+
+          # Schedule delayed_post transform command if needed
+          if config.delayed_post
+            Rubber.logger.info "Scheduling delayed_post: #{config.delayed_post}"
+            @delayed_post_commands << config.delayed_post
           end
         end
       end
@@ -191,6 +207,8 @@ module Rubber
       attr_accessor :write_cmd
       # The command to run after generating the config file if it has changed
       attr_accessor :post
+      # The command to run after generating _all_ config files if this file has changed.  Commands duplicated from multiple files are only run once
+      attr_accessor :delayed_post
       # The owner the output file should have, e.g. "root"
       attr_accessor :owner
       # The group the output file should have, e.g. "system"

data/lib/rubber/recipes/rubber/security_groups.rb

@@ -8,7 +8,7 @@ namespace :rubber do
   required_task :setup_security_groups do
     servers = find_servers_for_task(current_task)
 
-    cloud.setup_security_groups(servers.collect(&:host))
+    servers.collect(&:host).each{ |host| cloud.setup_security_groups(host) }
   end
 
   desc <<-DESC

data/lib/rubber/recipes/rubber/volumes.rb

@@ -132,7 +132,7 @@ namespace :rubber do
               fi
               mv /etc/fstab /etc/fstab.bak
               cat /etc/fstab.bak | grep -v '#{vol_spec['mount']}' > /etc/fstab
-              if [[ #{rubber_env.cloud_provider == 'aws'} && `lsb_release -r -s | sed 's/[.].*//'` -gt "10" ]]; then
+              if [[ #{rubber_env.cloud_provider == 'aws'} == true ]] && [[ `lsb_release -r -s | sed 's/[.].*//'` -gt "10" ]]; then
 		            device=`echo #{vol_spec['device']} | sed 's/sd/xvd/'`
 	            else
 		            device='#{vol_spec['device']}'

data/lib/rubber/version.rb

@@ -1,3 +1,3 @@
 module Rubber
-  VERSION = "2.6.5"
+  VERSION = "2.7.0"
 end

data/templates/delayed_job/config/rubber/deploy-delayed_job.rb

@@ -0,0 +1,44 @@
+namespace :rubber do
+ 
+  namespace :delayed_job do
+    
+    rubber.allow_optional_tasks(self)
+    
+    after "deploy:stop",    "rubber:delayed_job:stop"
+    after "deploy:start",   "rubber:delayed_job:start"
+    after "deploy:restart", "rubber:delayed_job:restart"
+    
+    def args
+      rubber_env.delayed_job_args || "-n #{rubber_env.num_delayed_job_workers}"
+    end
+ 
+    desc "Stop the delayed_job process"
+    task :stop, :roles => :delayed_job do
+      rsudo "cd #{current_path} && RAILS_ENV=#{Rubber.env} script/delayed_job stop #{self.args}", :as => rubber_env.app_user
+    end
+ 
+    desc "Start the delayed_job process"
+    task :start, :roles => :delayed_job do
+      rsudo "cd #{current_path} && RAILS_ENV=#{Rubber.env} script/delayed_job start #{self.args}", :as => rubber_env.app_user
+    end
+ 
+    desc "Restart the delayed_job process"
+    task :restart, :roles => :delayed_job do
+      rsudo "cd #{current_path} && RAILS_ENV=#{Rubber.env} script/delayed_job restart #{self.args}", :as => rubber_env.app_user
+    end
+ 
+    desc "Live tail of delayed_job log files for all machines"
+    task :tail_logs, :roles => :delayed_job do
+      last_host = ""
+      log_file_glob = rubber.get_env("FILE", "Log files to tail", true, "#{current_path}/log/delayed_job.log")
+      trap("INT") { puts 'Exiting...'; exit 0; }                    # handle ctrl-c gracefully
+      run "tail -qf #{log_file_glob}" do |channel, stream, data|
+        puts if channel[:host] != last_host                         # blank line between different hosts
+        host = "[#{channel.properties[:host].gsub(/\..*/, '')}]"    # get left-most subdomain
+        data.lines { |line| puts "%-15s %s" % [host, line] }        # add host name to the start of each line
+        last_host = channel[:host]
+        break if stream == :err
+      end
+    end
+  end
+end

data/templates/delayed_job/config/rubber/role/delayed_job/monit-delayed_job.conf

@@ -0,0 +1,18 @@
+<%
+  @path = '/etc/monit/monit.d/monit-delayed_job.conf'
+%>
+
+<% if rubber_env.num_delayed_job_workers == 1 %>
+check process delayed_job
+  with pidfile <%= Rubber.root %>/tmp/pids/delayed_job.pid
+  start program = "/usr/bin/sudo -H -u <%= rubber_env.app_user %> /bin/bash -l -c 'RAILS_ENV=<%= Rubber.env %> <%= Rubber.root %>/script/delayed_job start'"
+  stop program = "/usr/bin/sudo -H -u <%= rubber_env.app_user %> /bin/bash -l -c 'RAILS_ENV=<%= Rubber.env %> <%= Rubber.root %>/script/delayed_job stop'"
+<% else %>
+<% rubber_env.num_delayed_job_workers.times do |id| %>
+check process delayed_job_<%= id %>
+  with pidfile <%= Rubber.root %>/tmp/pids/delayed_job.<%= id %>.pid
+  start program = "/usr/bin/sudo -H -u <%= rubber_env.app_user %> /bin/bash -l -c 'RAILS_ENV=<%= Rubber.env %> <%= Rubber.root %>/script/delayed_job start -i <%= id %>'"
+  stop program = "/usr/bin/sudo -H -u <%= rubber_env.app_user %> /bin/bash -l -c 'RAILS_ENV=<%= Rubber.env %> <%= Rubber.root %>/script/delayed_job stop -i <%= id %>'"
+  group delayed_job
+<% end %>
+<% end %>

data/templates/delayed_job/config/rubber/rubber-delayed_job.yml

@@ -0,0 +1 @@
+num_delayed_job_workers: 1

data/templates/delayed_job/templates.yml

@@ -0,0 +1 @@
+description: The Delayed::Job module

data/templates/discourse/config/rubber/deploy-discourse.rb

@@ -0,0 +1,16 @@
+namespace :rubber do
+
+  namespace :discourse do
+
+    after "rubber:postgresql:bootstrap", "rubber:discourse:create_postgres_extensions"
+
+    task :create_postgres_extensions, :roles => :postgresql_master do
+      rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes install postgresql-contrib"
+      rubber.sudo_script "create_extensions", <<-ENDSCRIPT
+        sudo -i -u postgres psql -c "CREATE EXTENSION IF NOT EXISTS hstore;" -d "#{rubber_env.db_name}"
+        sudo -i -u postgres psql -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" -d "#{rubber_env.db_name}"
+      ENDSCRIPT
+    end
+  end
+
+end

data/templates/discourse/config/rubber/role/nginx/nginx.conf

@@ -0,0 +1,43 @@
+<%
+  @path = "/etc/nginx/nginx.conf"
+  @post = "mkdir -p #{rubber_env.nginx_log_dir}"
+%>
+
+user www-data;
+worker_processes 10;
+
+pid /var/run/nginx.pid;
+
+events
+{
+  worker_connections 1024;
+}
+
+http
+{
+  include           /etc/nginx/mime.types;
+  default_type      application/octet-stream;
+
+  sendfile          on;
+  tcp_nopush        on;
+  tcp_nodelay       off;
+
+  gzip              on;
+  gzip_http_version 1.0;
+  gzip_comp_level   2;
+  gzip_proxied      any;
+  gzip_min_length 1000;
+  gzip_types        application/json text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+  # configure log format like to Apache's "combined" log format
+  log_format        main
+                      '$remote_addr - $remote_user [$time_local] '
+                      '"$request" $status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_cookie"';
+
+  # default log files
+  error_log         <%= rubber_env.nginx_log_dir %>/error.log notice;
+  access_log        <%= rubber_env.nginx_log_dir %>/access.log main;
+
+  include /etc/nginx/rubber/*.conf;
+}

data/templates/discourse/config/rubber/role/nginx/unicorn_nginx.conf

@@ -0,0 +1,102 @@
+<%
+  @path = "/etc/nginx/rubber/unicorn_nginx.conf"
+%>
+
+upstream unicorn_server {
+ # This is the socket we configured in unicorn.rb
+ server unix:/var/run/unicorn.sock
+ fail_timeout=0;
+}
+
+server {
+  listen 80;
+  client_max_body_size 4G;
+  server_name <%= [ rubber_env.domain, rubber_env.web_aliases ].flatten.compact.join(" ") %>;
+
+  sendfile on;
+  keepalive_timeout 65;
+
+  # Location of our static files
+  root <%= Rubber.root + "/public" %>;
+  set $public <%= Rubber.root + "/public" %>;
+
+  location / {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+
+    # If you don't find the filename in the static files
+    # Then request it from the unicorn server
+    if (!-f $request_filename) {
+      proxy_pass http://unicorn_server;
+      break;
+    }
+
+    location ~ ^/assets/ {
+      expires 1y;
+      add_header Cache-Control public;
+      add_header ETag "";
+      break;
+    }
+
+    location ~ ^/uploads/ {
+      expires 1y;
+      add_header Cache-Control public;
+      add_header ETag "";
+
+      ## optional upload anti-hotlinking rules
+      #valid_referers none blocked mysite.com *.mysite.com;
+      #if ($invalid_referer) {
+      #  return 403;
+      #}
+
+      # let NGINX serve images
+      location ~* \.(gif|png|jpg|jpeg|bmp|tif|tiff)$ { try_files $uri =404; }
+      location ~ /_optimized/ { try_files $uri =404; }
+
+      # attachments must go through the rails application to get the right content-disposition header
+      proxy_set_header X-Sendfile-Type X-Accel-Redirect;
+      proxy_set_header X-Accel-Mapping $public/=/downloads/;
+      proxy_pass http://unicorn_server;
+      break;
+    }
+
+    try_files $uri @discourse;
+  }
+
+  location /downloads/ {
+    internal;
+    alias $public/;
+  }
+
+  location @discourse {
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $http_host;
+
+    proxy_pass http://unicorn_server;
+  }
+
+  # this rewrites all the requests to the maintenance.html
+  # page if it exists in the doc root. This is for capistrano's
+  # disable web task
+  if (-f $document_root/system/maintenance.html)
+  {
+    rewrite  ^(.*)$  /system/maintenance.html last;
+    break;
+  }
+
+  error_page   500 502 503 504  /500.html;
+  location = /500.html
+  {
+    root <%= Rubber.root + "/public" %>;
+  }
+  error_page 404  /404.html;
+  location = /404.html
+  {
+    root <%= Rubber.root + "/public" %>;
+  }
+}
+
+

data/templates/discourse/config/rubber/role/unicorn/unicorn.rb

@@ -0,0 +1,99 @@
+<%
+  @path = "#{Rubber.root}/config/unicorn.rb"
+  current_path = "/mnt/#{rubber_env.app_name}-#{Rubber.env}/current"
+%>
+
+worker_processes 2
+working_directory "<%= Rubber.root %>"
+
+# This loads the application in the master process before forking
+# worker processes
+# Read more about it here:
+# http://unicorn.bogomips.org/Unicorn/Configurator.html
+preload_app true
+
+timeout 30
+
+# This is where we specify the socket.
+# We will point the upstream Nginx module to this socket later on
+listen "/var/run/unicorn.sock", :backlog => 64
+
+pid "/var/run/unicorn.pid"
+
+# Set the path of the log files inside the log folder of the testapp
+stderr_path "<%= Rubber.root %>/log/unicorn.stderr.log"
+stdout_path "<%= Rubber.root %>/log/unicorn.stdout.log"
+
+# Because of Capistano, we need to tell unicorn where find the current Gemfile
+# Read about Unicorn, Capistrano, and Bundler here:
+# http://unicorn.bogomips.org/Sandbox.html
+before_exec do |server|
+  ENV['BUNDLE_GEMFILE'] = "<%= current_path %>/Gemfile"
+end
+
+initialized = false
+before_fork do |server, worker|
+  unless initialized
+    # load up the yaml for the localization bits, in master process
+    I18n.t(:posts)
+    # get rid of rubbish so we don't share it
+    GC.start
+  end
+  ActiveRecord::Base.connection.disconnect!
+  $redis.client.disconnect
+  ##
+  # When sent a USR2, Unicorn will suffix its pidfile with .oldbin and
+  # immediately start loading up a new version of itself (loaded with a new
+  # version of our app). When this new Unicorn is completely loaded
+  # it will begin spawning workers. The first worker spawned will check to
+  # see if an .oldbin pidfile exists. If so, this means we've just booted up
+  # a new Unicorn and need to tell the old one that it can now die. To do so
+  # we send it a QUIT.
+  #
+  # Using this method we get 0 downtime deploys.
+  old_pid = "/var/run/unicorn.pid.oldbin"
+  if File.exists?(old_pid) && server.pid != old_pid
+    begin
+      Process.kill("QUIT", File.read(old_pid).to_i)
+    rescue Errno::ENOENT, Errno::ESRCH
+      # someone else did our job for us
+    end
+  end
+
+  # This option works in together with preload_app true setting
+  # What is does is prevent the master process from holding
+  # the database connection
+  defined?(ActiveRecord::Base) and
+    ActiveRecord::Base.connection.disconnect!
+end
+
+
+after_fork do |server, worker|
+  # Unicorn master is started as root, which is fine, but let's
+  # drop the workers to www-data:www-data
+  begin
+    uid, gid = Process.euid, Process.egid
+    user, group = '<%=rubber_env.app_user %>', '<%=rubber_env.app_user %>'
+    target_uid = Etc.getpwnam(user).uid
+    target_gid = Etc.getgrnam(group).gid
+    worker.tmp.chown(target_uid, target_gid)
+    if uid != target_uid || gid != target_gid
+      Process.initgroups(user, target_gid)
+      Process::GID.change_privilege(target_gid)
+      Process::UID.change_privilege(target_uid)
+    end
+  rescue => e
+    if RAILS_ENV == 'development'
+      STDERR.puts "couldn't change user, oh well"
+    else
+      raise e
+    end
+  end
+
+  # Here we are establishing the connection after forking worker
+  # processes
+  ActiveRecord::Base.establish_connection
+  $redis.client.reconnect
+  Rails.cache.reconnect
+  MessageBus.after_fork
+end