rubber 1.15.0 → 2.0.0.pre1

data/templates/resque/config/rubber/deploy-resque.rb

@@ -0,0 +1,190 @@
+
+namespace :rubber do
+
+  namespace :resque do
+  
+    rubber.allow_optional_tasks(self)
+
+    namespace :worker do
+
+      rubber.allow_optional_tasks(self)
+
+      before "deploy:stop", "rubber:resque:worker:stop"
+      after "deploy:start", "rubber:resque:worker:start"
+      after "deploy:restart", "rubber:resque:worker:restart"
+
+      desc "Starts resque workers"
+      task :start, :roles => :resque_worker do
+        rsudo "service resque-pool start"
+      end
+
+      desc "Stops resque workers and monitor"
+      task :stop, :roles => :resque_worker do
+        rsudo "service resque-pool stop || true"
+      end
+
+      desc "Force kill all resque workers and monitor"
+      task :force_stop, :roles => :resque_worker do
+        rsudo "kill -TERM `cat #{rubber_env.resque_pool_pid_file}`"
+      end
+
+      desc "Restarts resque workers"
+      task :restart, :roles => :resque_worker do
+        stop
+        start
+      end
+
+      desc "Continuously show worker stats"
+      task :stats, :roles => :resque_worker do
+        logger.level = 0
+
+        WorkerItem = Struct.new(:host, :pid, :process_type, :target, :start_time)
+        WorkerItem.class_eval do
+          def uid
+            "#{host}:#{pid}"
+          end
+        end
+
+        mutex = Mutex.new
+
+        while true do
+
+          host_data = {}
+          run "ps ax | grep '[r]esque.*:'; exit 0" do |channel, stream, data|
+            if data
+              host = channel.properties[:host].gsub(/\..*/, '')
+              mutex.synchronize do
+                host_data[host] ||= ""
+                host_data[host] << data
+              end
+            end
+          end
+
+          queue_sizes = capture 'redis-cli --raw smembers resque:queues | while read x; do if [[ -z $x ]]; then continue; fi; echo -n "$x "; echo "llen resque:queue:$x" | redis-cli --raw; done', :roles => :redis
+          queue_sizes = Hash[*queue_sizes.split]
+
+          idle = []
+          starting = []
+          paused = []
+          parents = []
+          children = []
+
+          host_data.each do |host, data|
+            data.lines do |line|
+              line.chomp!
+              cols = line.split
+
+              # 12887 ?        Sl     0:01 resque-1.13.0: Forked 31395 at 1300564431
+              # 13054 ?        Sl     0:23 resque-1.13.0: Processing facebook since 1300565337
+              # 28561 ?        Sl     0:03 resque-1.13.0: Waiting for *index*
+
+              item = WorkerItem.new
+              item.host = host
+              item.pid = cols[0]
+              item.process_type = case cols[5]
+                when 'Processing' then :child
+                when 'Forked' then :parent
+                when 'Waiting' then (cols.delete_at(6); :idle)
+                when 'Starting' then :starting
+                when 'Paused' then :paused
+                else :unknown
+              end
+              item.target = cols[6]
+              item.start_time = cols[8].to_i
+
+              idle << item if item.process_type == :idle
+              starting << item if item.process_type == :starting
+              paused << item if item.process_type == :paused
+              parents << item if item.process_type == :parent
+              children << item if item.process_type == :child
+            end
+          end
+
+          pairs = {}
+          [*parents, *children].each {|item| pairs[item.uid] ||= [];  pairs[item.uid] << item}
+          stuck_parents = pairs.select{|item| item.size == 1 && item.first.type == :parent}
+          stuck_children = pairs.select{|item| item.size == 1 && item.first.type == :child}
+
+          print "\e[H\e[2J"
+          puts Time.now
+          puts ""
+
+          counts = children.group_by {|item| item.target || 'unknown' }
+          queue_sizes.each {|target, count| counts[target] ||= [] }
+
+          fmt = "%-37s %-7s"
+          puts fmt % ["Working", counts.values.collect(&:size).inject(0) { |sum, p| sum + p }]
+          puts fmt % ["Idle", idle.size]
+          puts fmt % ["Starting", starting.size]
+          puts fmt % ["Paused", paused.size]
+          puts fmt % ["Stuck Parents", stuck_parents.size]
+          puts fmt % ["Orphans", stuck_children.size]
+
+          puts ""
+          fmt = "%-37s %-10s %-10s"
+          puts fmt % %w{Queue Working Queued}
+          counts.sort.each do |target, items|
+            working = items.size
+            queued = queue_sizes[target].to_i
+            if working > 0 || queued > 0
+              puts fmt % [target, working, queued]
+            end
+          end
+
+          puts ""
+          fmt = "%-10s %-6s"
+          puts fmt % %w{Runtime Count}
+
+          times = [1, 5, 15, 30, 60, 120]
+          slow_times = [180, 540]
+          ages = children.group_by do |item|
+            runtime = Time.now.to_i - item.start_time
+            runtime = 0 if item.start_time == 0
+            [*times, *slow_times].find {|t| runtime < (t * 60)}
+          end
+
+          times.each do |t|
+            items = ages[t] || []
+            puts fmt % [t, items.size]
+          end
+          slow_times.each do |t|
+            items = ages[t] || []
+            slow_queues = items.collect(&:target).sort.uniq.join(', ')
+            puts "#{fmt} %s" % ["#{t}", items.size, slow_queues]
+          end
+          slow_queues = (ages[nil] || []).collect(&:target).sort.uniq.join(', ')
+          puts "#{fmt} %s" % [">#{slow_times.last}", (ages[nil] || []).size, slow_queues]
+
+          sleep 10
+        end
+      end
+      
+    end
+
+    namespace :web do
+      rubber.allow_optional_tasks(self)
+      
+      before "deploy:stop", "rubber:resque:web:stop"
+      after "deploy:start", "rubber:resque:web:start"
+      after "deploy:restart", "rubber:resque:web:restart"
+
+      desc "Starts resque web tools"
+      task :start, :roles => :resque_web do
+        rsudo "service resque-web start"
+      end
+
+      desc "Stops resque web tools"
+      task :stop, :roles => :resque_web do
+        rsudo "service resque-web stop || true"
+      end
+
+      desc "Restarts resque web tools"
+      task :restart, :roles => :resque_web do
+        stop
+        start
+      end
+
+    end
+
+  end
+end

data/templates/resque/config/rubber/role/resque_web/resque-web-upstart.conf

@@ -0,0 +1,20 @@
+<%
+  @path = "/etc/init/resque-web.conf"
+  @backup = false
+%>
+description "resque web server"
+
+start on started redis
+stop on runlevel [016]
+
+script
+  exec sudo -u <%= rubber_env.app_user %> /bin/bash -l -c 'cd <%= Rubber.root %> && exec bundle exec resque-web --foreground --port <%= rubber_env.resque_web_port %> --no-launch --app-dir tmp/resque_web config/initializers/resque.rb > log/resque-web.log 2>&1'
+end script
+
+post-start script
+  status resque-web | head -n1 | awk '{print $NF}' > <%= rubber_env.resque_web_pid_file %>
+end script
+
+post-stop script
+  rm -f <%= rubber_env.resque_web_pid_file %>
+end script

data/templates/resque/config/rubber/role/resque_worker/resque-pool-upstart.conf

@@ -0,0 +1,50 @@
+<%
+  @path = "/etc/init/resque-pool.conf"
+  @backup = false
+%>
+description "resque pool server"
+
+start on started redis
+stop on runlevel [016]
+
+# not available in upstart 0.6.5
+# kill signal INT
+
+script
+  cd <%= Rubber.root %>
+  exec sudo -u <%= rubber_env.app_user %> /bin/bash -l -c 'cd <%= Rubber.root %> && exec bundle exec resque-pool > log/resque-pool.log 2>&1'
+end script
+
+post-start script
+  status resque-pool | head -n1 | awk '{print $NF}' > <%= rubber_env.resque_pool_pid_file %>
+end script
+
+# using pre-stop prevents restarts in upstart on ubuntu 10.04,
+# but no choice since "kill signal" not available in that ver
+# of upstart, and we need to kill INT on restart during deploy
+# so that we dont end up killing running jobs
+#
+pre-stop script
+  pid=`cat /var/run/resque-pool.pid`
+  kill -INT $pid
+  while ps -p $pid > /dev/null; do sleep 0.1; done
+  rm -f <%= rubber_env.resque_pool_pid_file %>
+end script
+
+# The pool manager responds to the following signals:
+#
+# HUP - reload the config file, reload logfiles, restart all workers.
+# QUIT - send QUIT to each worker parent and shutdown the manager after all workers are done.
+# INT - send QUIT to each worker parent and immediately shutdown manager
+# TERM - send TERM to each worker parent and immediately shutdown manager
+# WINCH - send QUIT to each worker, but keep manager running (send HUP to reload config and restart workers)
+# USR1/USR2/CONT - pass the signal on to all worker parents (see Resque docs).
+# Use HUP to help logrotate run smoothly and to change the number of workers per worker type.
+#
+# Resque workers respond to a few different signals:
+#
+# QUIT - Wait for child to finish processing then exit
+# TERM / INT - Immediately kill child then exit
+# USR1 - Immediately kill child but don't exit
+# USR2 - Don't start to process any new jobs
+# CONT - Start to process new jobs again after a USR2

data/templates/resque/config/rubber/role/resque_worker/resque-pool.yml

@@ -0,0 +1,6 @@
+<%
+  @path = "#{Rubber.root}/config/resque-pool.yml"
+  worker_count = rubber_env.resque_worker_count.to_i
+%>
+
+'*': <%= rubber_env.resque_worker_count.to_i %>

data/{generators/vulcanize/templates → templates}/resque/config/rubber/rubber-resque.yml

@@ -1,8 +1,12 @@
 resque_web_port: 5678
+resque_web_pid_file: /var/run/resque-web.pid
+resque_pool_pid_file: /var/run/resque-pool.pid
 
 # list of workers with the queues to be run on each - on each worker instance,
 # a worker is run for each item in this list to handle the queues specified
 #
+resque_worker_count: "#{Rubber.env == 'production' ? 4 : 2}"
+
 resque_workers:
   -
     queues: "*"
@@ -11,7 +15,13 @@ resque_workers:
     queues: "*"
     poll_interval: 1
 
-gems: [yajl-ruby resque]
+gems: [yajl-ruby, resque, resque-pool]
+
+web_tools_proxies:
+  resque:
+    role: resque_web
+    port: "#{resque_web_port}"
+    path: /
 
 role_dependencies:
   web_tools: [resque_web]

data/templates/resque/lib/tasks/resque.rake

@@ -0,0 +1,18 @@
+require 'resque/pool/tasks'
+
+# this task will get called before resque:pool:setup
+# and preload the rails environment in the pool manager
+task "resque:setup" => :environment do
+  # generic worker setup, e.g. Hoptoad for failed jobs
+end
+
+task "resque:pool:setup" do
+  # close any sockets or files in pool manager
+  ActiveRecord::Base.connection.disconnect!
+  
+  # and re-open them in the resque worker parent
+  Resque::Pool.after_prefork do |job|
+    ActiveRecord::Base.establish_connection
+    Resque.redis = YAML.load_file(Rails.root.join('config', 'resque.yml'))[Rails.env]
+  end
+end

data/templates/resque/templates.rb

@@ -0,0 +1,2 @@
+append_to_file 'Gemfile', "gem 'resque'\n" if Rubber::Util::is_bundler?
+append_to_file 'Gemfile', "gem 'resque-pool'\n" if Rubber::Util::is_bundler?

data/{lib/generators/vulcanize/templates → templates}/sphinx/config/rubber/common/sphinx.yml

@@ -1,10 +1,10 @@
 <%
-  @path = "#{RUBBER_ROOT}/config/sphinx.yml"
+  @path = "#{Rubber.root}/config/sphinx.yml"
 %>
 
 # Taken from: http://ts.freelancing-gods.com/rdoc/classes/ThinkingSphinx/Configuration.html
 #
-# Here are the default settings, relative to RUBBER_ROOT where relevant:
+# Here are the default settings, relative to Rubber.root where relevant:
 #
 # config file::           config/#{environment}.sphinx.conf
 # searchd log file::      log/searchd.log
@@ -36,11 +36,11 @@
 # Each setting in the YAML file is optional - so only put in the ones you
 # want to change.
 
-<%= RUBBER_ENV %>:
+<%= Rubber.env %>:
   address:            <%= rubber_instances.for_role('sphinx').first.full_name rescue 'localhost' %>
-  mem_limit:          <%= RUBBER_ENV == 'production' ? '1024M' : '64M' %>
-  config_file:        sphinx/config/<%= RUBBER_ENV %>.sphinx.conf
-  searchd_file_path:  sphinx/db/<%= RUBBER_ENV %>/
-  pid_file:           <%= RUBBER_ROOT %>/log/searchd.<%= RUBBER_ENV %>.pid
-  searchd_log_file:   <%= RUBBER_ROOT %>/log/searchd.log
-  query_log_file:     <%= RUBBER_ROOT %>/log/searchd.query.log
+  mem_limit:          <%= Rubber.env == 'production' ? '1024M' : '64M' %>
+  config_file:        sphinx/config/<%= Rubber.env %>.sphinx.conf
+  searchd_file_path:  sphinx/db/<%= Rubber.env %>/
+  pid_file:           <%= Rubber.root %>/log/searchd.<%= Rubber.env %>.pid
+  searchd_log_file:   <%= Rubber.root %>/log/searchd.log
+  query_log_file:     <%= Rubber.root %>/log/searchd.query.log

data/{generators/vulcanize/templates → templates}/sphinx/config/rubber/deploy-sphinx.rb

@@ -72,7 +72,7 @@ namespace :rubber do
 
     # runs the given ultrasphinx rake tasks
     def run_sphinx task
-      rsudo "cd #{current_path} && RAILS_ENV=#{RUBBER_ENV} #{fetch(:rake, 'rake')} #{task}", :as => runner
+      rsudo "cd #{current_path} && RAILS_ENV=#{Rubber.env} #{fetch(:rake, 'rake')} #{task}", :as => runner
     end
 
 

data/templates/sphinx/config/rubber/role/sphinx/crontab

@@ -0,0 +1,11 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ["# rubber-sphinx-start", "# rubber-sphinx-end"]
+%>
+
+# incremental index once every 10 minutes
+*/10 * * * * <%= Rubber.root %>/script/rubber cron --user <%= rubber_env.app_user %> --rake ts:in:delta
+
+# full reindex daily at 4:35
+35 4 * * * <%= Rubber.root %>/script/rubber cron --user <%= rubber_env.app_user %> --rake ts:index

data/templates/unicorn/config/rubber/deploy-unicorn.rb

@@ -0,0 +1,35 @@
+
+namespace :rubber do
+
+  namespace :unicorn do
+  
+    rubber.allow_optional_tasks(self)
+    
+    before "deploy:stop", "rubber:unicorn:stop"
+    after "deploy:start", "rubber:unicorn:start"
+    after "deploy:restart", "rubber:unicorn:reload"
+    
+    desc "Stops the unicorn server"
+    task :stop, :roles => :unicorn do
+      rsudo "if [ -f /var/run/unicorn.pid ]; then pid=`cat /var/run/unicorn.pid` && kill -TERM $pid; fi"
+    end
+    
+    desc "Starts the unicorn server"
+    task :start, :roles => :unicorn do
+      rsudo "cd #{current_path} && bundle exec unicorn_rails -c #{current_path}/config/unicorn.rb -E #{Rubber.env} -D"
+    end
+    
+    desc "Restarts the unicorn server"
+    task :restart, :roles => :unicorn do
+      stop
+      start
+    end
+  
+    desc "Reloads the unicorn web server"
+    task :reload, :roles => :unicorn do
+      rsudo "if [ -f /var/run/unicorn.pid ]; then pid=`cat /var/run/unicorn.pid` && kill -USR2 $pid; else cd #{current_path} && bundle exec unicorn_rails -c #{current_path}/config/unicorn.rb -E #{Rubber.env} -D; fi"
+    end
+  
+  end
+
+end

data/templates/unicorn/config/rubber/role/nginx/unicorn_nginx.conf

@@ -0,0 +1,56 @@
+<%
+  @path = "/etc/nginx/rubber/unicorn_nginx.conf"
+%>
+
+
+upstream unicorn_server {
+ # This is the socket we configured in unicorn.rb
+ server unix:/var/run/unicorn.sock
+ fail_timeout=0;
+}
+
+server {
+  listen 80;
+  client_max_body_size 4G;
+  server_name <%= [ rubber_env.domain, rubber_env.web_aliases ].flatten.compact.join(" ") %>;
+
+  keepalive_timeout 5;
+
+  # Location of our static files
+  root <%= Rubber.root + "/public" %>;
+
+  location / {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+
+    # If you don't find the filename in the static files
+    # Then request it from the unicorn server
+    if (!-f $request_filename) {
+      proxy_pass http://unicorn_server;
+      break;
+    }
+  }
+
+	# this rewrites all the requests to the maintenance.html
+	# page if it exists in the doc root. This is for capistrano's
+	# disable web task
+	if (-f $document_root/system/maintenance.html)
+	{
+	  rewrite  ^(.*)$  /system/maintenance.html last;
+	  break;
+	}
+
+	error_page   500 502 503 504  /500.html;
+	location = /500.html
+	{
+	  root <%= Rubber.root + "/public" %>;
+	}
+	error_page 404  /404.html;
+	location = /404.html
+	{
+	  root <%= Rubber.root + "/public" %>;
+	}
+}
+
+