capistrano-kitchen 0.0.0.pre

data/lib/capistrano_kitchen/dishes/nginx_unicorn/hooks.rb

@@ -0,0 +1,11 @@
+# @author Donovan Bray <donnoman@donovanbray.com>
+
+Capistrano::Configuration.instance(true).load do
+  after "deploy:provision", "nginx_unicorn:install"
+  after "deploy:setup", "nginx_unicorn:setup"
+  after "logrotate:rotate", "nginx_unicorn:reopen"
+  after "sdagent:setup", "nginx_unicorn:setup_sdagent"
+  after "nginx_unicorn:install", "nginx_unicorn:setup"
+  after "nginx_unicorn:setup", "nginx_unicorn:configure"
+  on :load, "nginx_unicorn:watcher"
+end

data/lib/capistrano_kitchen/dishes/nginx_unicorn/install.rb

@@ -0,0 +1,176 @@
+# @author Donovan Bray <donnoman@donovanbray.com>
+require File.expand_path(File.dirname(__FILE__) + '/../utilities')
+
+# This Nginx is targeted for the :app role meant to be acting as a front end
+# to a unicorn based application
+
+# Additions
+# https://github.com/newobj/nginx-x-rid-header
+# https://github.com/yaoweibin/nginx_syslog_patch
+
+# Possible Future Additions
+# https://support.newrelic.com/kb/features/tracking-front-end-time
+
+Capistrano::Configuration.instance(true).load do
+
+  namespace :nginx_unicorn do
+    set :nginx_unicorn_init_d, "nginx_unicorn"
+    set :nginx_unicorn_root, "/opt/nginx_unicorn"
+    set :nginx_unicorn_conf_path, File.join(File.dirname(__FILE__),'nginx.conf')
+    set(:nginx_unicorn_conf_dir) {"#{nginx_unicorn_root}/conf"}
+    set :nginx_unicorn_init_d_path, File.join(File.dirname(__FILE__),'nginx_unicorn.init')
+    set :nginx_unicorn_stub_conf_path, File.join(File.dirname(__FILE__),'stub_status.conf')
+    set :nginx_unicorn_god_path, File.join(File.dirname(__FILE__),'nginx_unicorn.god')
+    set :nginx_unicorn_logrotate_path, File.join(File.dirname(__FILE__),'nginx_unicorn.logrotate')
+    set :nginx_unicorn_mime_types_erb, File.join(File.dirname(__FILE__),'mime.types.erb')
+    # must be above 1.1.7 http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2012-1180
+    set :nginx_unicorn_src, "http://nginx.org/download/nginx-1.2.0.tar.gz"
+    set(:nginx_unicorn_ver) { nginx_unicorn_src.match(/\/([^\/]*)\.tar\.gz$/)[1] }
+    set(:nginx_unicorn_source_dir) {"#{nginx_unicorn_root}/src/#{nginx_unicorn_ver}"}
+    set(:nginx_unicorn_patch_dir) {"#{nginx_unicorn_root}/src"}
+    set(:nginx_unicorn_upstream_socket){"#{shared_path}/sockets/unicorn.sock"}
+    set(:nginx_unicorn_log_dir) {"#{nginx_unicorn_root}/logs"}
+    set(:nginx_unicorn_pid_file) {"#{nginx_unicorn_log_dir}/nginx.pid"}
+    set(:nginx_unicorn_sbin_file) {"#{nginx_unicorn_root}/sbin/nginx"}
+    set :nginx_unicorn_watcher, nil
+    set :nginx_unicorn_user, "nobody"
+    set :nginx_unicorn_suppress_runner, false
+    set :nginx_unicorn_port, '80'
+    set :nginx_unicorn_server_name, 'localhost'
+    set :nginx_unicorn_app_conf_path, File.join(File.dirname(__FILE__),'app.conf')
+    set :nginx_unicorn_set_scheme, true
+    set :nginx_unicorn_worker_processes, "1" # should be cpu's - 1
+    set :nginx_unicorn_gzip, true
+    set :nginx_unicorn_fail_timeout, nil
+    set :nginx_unicorn_syslog_patch, true
+    set :nginx_unicorn_rid_header_patch, false # while we want this to be true by default it makes the configurations incompatible
+                                               # with the previous default. Which can cause a working NGINX to stop working until recompiled.
+    set :nginx_unicorn_use_503_instead_of_502, false # useful if you are behind a load balancer that only understands 503's.
+
+    set(:nginx_unicorn_configure_flags) {[
+      "--prefix=#{nginx_unicorn_root}",
+      "--sbin-path=#{nginx_unicorn_sbin_file}",
+      "--pid-path=#{nginx_unicorn_pid_file}",
+      "--conf-path=#{nginx_unicorn_conf_dir}/nginx.conf",
+      "--with-debug",
+      "--with-http_gzip_static_module",
+      "--with-http_stub_status_module",
+      "--with-http_ssl_module",
+      "--with-ld-opt=-lossp-uuid",
+      "--with-cc-opt=-I/usr/include/ossp"
+    ]}
+
+    desc "select watcher"
+    task :watcher do
+      nginx_unicorn.send("watch_with_#{nginx_unicorn_watcher}".to_sym) unless nginx_unicorn_watcher.nil?
+    end
+
+    desc "Use GOD as nginx_unicorn's runner"
+    task :watch_with_god do
+      #rejigger the maintenance tasks to use god when god is in play
+      %w(start stop restart).each do |t|
+        task t.to_sym, :roles => :app do
+          god.cmd "#{t} nginx_unicorn" unless nginx_unicorn_suppress_runner
+        end
+      end
+      after "god:setup", "nginx_unicorn:setup_god"
+    end
+
+    desc "setup god to watch nginx_unicorn"
+    task :setup_god, :roles => :app do
+      god.upload nginx_unicorn_god_path, 'nginx_unicorn.god'
+    end
+
+    desc 'Installs nginx for unicorn'
+    task :install, :roles => :app do
+      utilities.apt_install "libssl-dev zlib1g-dev libcurl4-openssl-dev libpcre3-dev libossp-uuid-dev git-core"
+      sudo "mkdir -p #{nginx_unicorn_source_dir}"
+      run "cd #{nginx_unicorn_root}/src && #{sudo} wget --tries=2 -c --progress=bar:force #{nginx_unicorn_src} && #{sudo} tar zxvf #{nginx_unicorn_ver}.tar.gz"
+      if nginx_unicorn_syslog_patch
+        nginx_unicorn_configure_flags << "--add-module=#{nginx_unicorn_patch_dir}/nginx_syslog_patch"
+        utilities.git_clone_or_pull "git://github.com/yaoweibin/nginx_syslog_patch.git", "#{nginx_unicorn_patch_dir}/nginx_syslog_patch"
+        run "cd #{nginx_unicorn_source_dir} && #{sudo} sh -c 'patch -p1 < #{nginx_unicorn_patch_dir}/nginx_syslog_patch/syslog_#{nginx_unicorn_ver.split('-').last}.patch'"
+      end
+      if nginx_unicorn_rid_header_patch
+        nginx_unicorn_configure_flags << "--add-module=#{nginx_unicorn_patch_dir}/nginx-x-rid-header"
+        utilities.git_clone_or_pull "git://github.com/newobj/nginx-x-rid-header.git", "#{nginx_unicorn_patch_dir}/nginx-x-rid-header"
+      end
+      run "cd #{nginx_unicorn_source_dir} && #{sudo} ./configure #{nginx_unicorn_configure_flags.join(" ")} && #{sudo} make"
+      run "cd #{nginx_unicorn_source_dir} && #{sudo} make install"
+    end
+
+    task :setup, :roles => :app do
+      sudo "mkdir -p #{nginx_unicorn_conf_dir}/sites-available #{nginx_unicorn_conf_dir}/sites-enabled #{nginx_unicorn_log_dir}"
+      utilities.sudo_upload_template nginx_unicorn_conf_path,"#{nginx_unicorn_conf_dir}/nginx.conf"
+      utilities.sudo_upload_template nginx_unicorn_mime_types_erb,"#{nginx_unicorn_conf_dir}/mime.types"
+      utilities.sudo_upload_template nginx_unicorn_stub_conf_path,"#{nginx_unicorn_conf_dir}/sites-available/stub_status.conf"
+      sudo "ln -sf #{nginx_unicorn_conf_dir}/sites-available/stub_status.conf #{nginx_unicorn_conf_dir}/sites-enabled/stub_status.conf"
+      utilities.sudo_upload_template nginx_unicorn_init_d_path,"/etc/init.d/#{nginx_unicorn_init_d}", :mode => "u+x"
+      utilities.sudo_upload_template nginx_unicorn_logrotate_path,"/etc/logrotate.d/#{nginx_unicorn_init_d}"
+    end
+
+    desc "Nginx Unicorn Reload"
+    task :reload, :roles => :app do
+      sudo "/etc/init.d/#{nginx_unicorn_init_d} reload"
+    end
+
+    desc "Nginx Unicorn Reopen"
+    task :reopen, :roles => :app do
+      sudo "/etc/init.d/#{nginx_unicorn_init_d} reopen"
+    end
+
+    task :remove_default, :roles => :app do
+      sudo "rm -f #{nginx_unicorn_conf_dir}/sites-enabled/default"
+    end
+
+    desc "Watch Nginx and Unicorn Workers with GOD"
+    task :setup_god, :roles => :app do
+      god.upload nginx_unicorn_god_path, "#{nginx_unicorn_init_d}.god"
+      # disable init from automatically starting and stopping these init controlled apps
+      # god will be started by init, and in turn start these god controlled apps.
+      # but leave the init script in place to be called manually
+      sudo "update-rc.d -f nginx_unicorn remove; true"
+      #if you simply remove lsb driven links an apt-get can later reinstall them
+      #so we explicitly define the kill scripts.
+      sudo "update-rc.d nginx_unicorn stop 20 2 3 4 5 .; true"
+    end
+
+    desc "Setup sd-agent to collect metrics for nginx"
+    task :setup_sdagent, :roles => :app do
+      # block executing this task if :sdagent isn't present on any :app servers.
+      if (find_servers(:roles => :app).map{|d| d.host} & find_servers(:roles => :sdagent).map{|d| d.host}).any?
+        sudo "sed -i 's/^.*nginx_status_url.*$/nginx_status_url: http:\\/\\/127.0.0.1\\/nginx_status/g' #{sdagent_root}/config.cfg"
+      end
+    end
+
+    desc "Write the application conf"
+    task :configure, :roles => :app do
+      utilities.sudo_upload_template nginx_unicorn_app_conf_path, "#{nginx_unicorn_conf_dir}/sites-available/#{application}.conf"
+      enable
+    end
+
+    desc "remove the application conf"
+    task :deconfigure, :roles => :app do
+      disable
+      sudo "rm -rf #{nginx_unicorn_conf_dir}/sites-available/#{application}.conf"
+    end
+
+    desc "Enable the application conf"
+    task :enable, :roles => :app do
+      sudo "ln -sf #{nginx_unicorn_conf_dir}/sites-available/#{application}.conf #{nginx_unicorn_conf_dir}/sites-enabled/#{application}.conf"
+    end
+
+    desc "Disable the application conf"
+    task :disable, :roles => :app do
+      sudo "rm -f #{nginx_unicorn_conf_dir}/sites-enabled/#{application}.conf"
+    end
+
+    %w(start stop restart).each do |t|
+      desc "#{t} nginx_unicorn via init"
+      task t.to_sym, :roles => :app do
+        sudo "/etc/init.d/#{nginx_unicorn_init_d} #{t}" unless nginx_unicorn_suppress_runner
+      end
+    end
+
+  end
+end

data/lib/capistrano_kitchen/dishes/nginx_unicorn/manage.rb

@@ -0,0 +1 @@
+# no longer required, all methods moved to install.rb

data/lib/capistrano_kitchen/dishes/nginx_unicorn/mime.types.erb

@@ -0,0 +1,79 @@
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/x-javascript              js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg svgz;
+    image/webp                            webp;
+
+    application/java-archive              jar war ear;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/xhtml+xml                 xhtml;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              eot;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/ogg                             ogg;
+    audio/x-m4a                           m4a;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mp4                             mp4;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/webm                            webm;
+    video/x-flv                           flv;
+    video/x-m4v                           m4v;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

data/lib/capistrano_kitchen/dishes/nginx_unicorn/nginx.conf

@@ -0,0 +1,138 @@
+#user  <%=nginx_unicorn_user%>;
+worker_processes  <%=nginx_unicorn_worker_processes%>;
+
+# nginx  file limits.
+worker_rlimit_nofile 40000;
+
+# worker_connections specifies how many network connections a worker is
+# allowed to maintain. worker_rlimit_nofile specifies how many open file
+# handles are allowed per worker. Since all tcp connections are file
+# handles (descriptors) on *nix systems, worker_rlimit_nofile must be
+# greater than worker_connections.
+
+<% if nginx_unicorn_syslog_patch %>
+syslog local6 <%=nginx_unicorn_init_d%>;
+error_log syslog:info|<%=nginx_unicorn_log_dir%>/error.log;
+<% else %>
+error_log <%=nginx_unicorn_log_dir%>/error.log;
+<% end %>
+
+# pid <%=nginx_unicorn_pid_file%>;
+
+events {
+    worker_connections  30000;
+    use epoll;
+}
+
+http {
+    include       mime.types;
+    types {
+      audio/x-wav wav;
+    }
+    default_type  application/octet-stream;
+
+    # hide the server version
+    server_tokens off;
+
+    # ssl_certificate <%=nginx_unicorn_root%>/wildcard.homerun.com.crt;
+    # ssl_certificate_key <%=nginx_unicorn_root%>/wildcard.homerun.com.key;
+    ssl_protocols        SSLv3 TLSv1;
+
+    # http://matt.io/technobabble/hivemind_devops_alert:_nginx_does_not_suck_at_ssl/ur
+    # http://news.ycombinator.com/item?id=2759596
+    # You can force nginx to not enable the expensive cipher by excluding all DHE ciphers.
+    # Add "!kEDH" to your cipher list. It disables (the ! disables) any cipher using
+    # Ephemeral Diffie-Hellman.
+    ssl_ciphers          HIGH:!ADH:!MD5:!kEDH;
+
+    server_names_hash_bucket_size 128;
+
+    # The proxy_read_timeout directive sets the read timeout for the response of the proxied server,
+    # in seconds by default. It determines how long nginx will wait to get the response to a request.
+    # The timeout is established not for entire response, but only between two operations of reading.
+    proxy_read_timeout 121s;
+
+    # $proxy_add_x_forwarded_for
+    # Contains client request-header "X-Forwarded-For" with separated by comma $remote_addr.
+    # If there is no X-Forwarded-For request-header, than $proxy_add_x_forwarded_for is equal to $remote_addr.
+
+<% if nginx_unicorn_rid_header_patch -%>
+    # For nginx_unicorn we should use the X-Request-Id header passed in on the connection
+    # On your backend, you can pull the request header x-exampledotcom-rid, and log it or tie it to whatever you may like.
+    # This makes it really easy to correlate backend exceptions or instrumentation with frontend http request logs.
+    # Goes well with the new Rails 3.2/master support for X-Request-Id.
+    proxy_set_header X-Request-Id $request_id;
+
+    log_format timing '$remote_addr - $remote_user [$time_local] "$http_user_agent" $scheme $http_host "$request" $status - request_time $request_time upstream_response_time $upstream_response_time upstream_cache_status $upstream_cache_status request_id "$request_id"';
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" - $connection $request_time $upstream_cache_status "$request_id"';
+<% else -%>
+    log_format timing '$remote_addr - $remote_user [$time_local] "$http_user_agent" $scheme $http_host "$request" $status - request_time $request_time upstream_response_time $upstream_response_time upstream_cache_status $upstream_cache_status';
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" - $connection $request_time $upstream_cache_status';
+<% end -%>
+
+
+    client_body_temp_path   '/dev/shm';
+    server_name_in_redirect on;
+    ignore_invalid_headers  on;
+
+    #default: keepalive_timeout 75
+    keepalive_timeout       75 20;
+
+    #default: keepalive_requests 100
+    keepalive_requests      20;
+    sendfile                on;
+    tcp_nodelay             on;
+    tcp_nopush              on;
+<% if nginx_unicorn_gzip %>
+    gzip  on;
+    gzip_http_version 1.0;
+    gzip_min_length 0;
+    gzip_buffers 16 8k;
+    gzip_comp_level 6;
+    gzip_static on;
+    gzip_proxied any;
+    gzip_vary on;
+    gzip_types text/plain text/javascript text/css application/x-javascript text/xml;
+<% end %>
+    ##
+    # Optimizations: http://www.typemiss.net/blog/kounoike/20060227-75
+    client_header_timeout         10m;
+    client_body_timeout           10m;
+    send_timeout                  10m;
+
+    connection_pool_size          256;
+    client_header_buffer_size     12k;
+    large_client_header_buffers   4 8k;
+    request_pool_size             4k;
+
+    output_buffers                1 32k;
+    postpone_output               1460;
+
+    # proxy_buffers
+    # syntax: proxy_buffers the_number is_size;
+    # default: proxy_buffers 8 4k/8k;
+    # context: http, server, location
+    # This directive sets the number and the size of buffers, into which will be read the answer, obtained from the proxied server.
+    # By default, the size of one buffer is equal to the size of page. Depending on platform this is either 4K or 8K.
+
+    proxy_buffers   16 32k;
+
+    # proxy_buffer_size
+    # syntax: proxy_buffer_size the_size;
+    # default: proxy_buffer_size 4k/8k;
+    # context: http, server, location
+    # This directive set the buffer size, into which will be read the first part of the response, obtained from the proxied server.
+    # In this part of response the small response-header is located, as a rule.
+    # By default, the buffer size is equal to the size of one buffer in directive proxy_buffers; however, it is possible to set it to less.
+
+    # proxy_busy_buffers_size
+    # syntax: proxy_busy_buffers_size size;
+    # default: proxy_busy_buffers_size proxy_buffer_size * 2;
+    # context: http, server, location, if
+    # TODO: Description.
+
+    include <%=nginx_unicorn_conf_dir%>/sites-enabled/*;
+
+}

data/lib/capistrano_kitchen/dishes/nginx_unicorn/nginx_unicorn.god

@@ -0,0 +1,47 @@
+God.watch do |w|
+  w.name = "<%=nginx_unicorn_init_d%>"
+  w.group = "nginx"
+  w.interval = 5.seconds # default
+  w.start = "/etc/init.d/<%=nginx_unicorn_init_d%> start"
+  w.stop = "/etc/init.d/<%=nginx_unicorn_init_d%> stop"
+  w.restart = "/etc/init.d/<%=nginx_unicorn_init_d%> restart"
+  w.pid_file = "<%=nginx_unicorn_pid_file%>"
+
+  # clean pid files before start if necessary
+  w.behavior(:clean_pid_file)
+
+  # determine the state on startup
+  w.transition(:init, { true => :up, false => :start }) do |on|
+    on.condition(:process_running) do |c|
+      c.running = true
+    end
+  end
+
+  # determine when process has finished starting
+  w.transition([:start, :restart], :up) do |on|
+    on.condition(:process_running) do |c|
+      c.running = true
+    end
+  end
+
+  # start if process is not running
+  w.transition(:up, :start) do |on|
+    on.condition(:process_exits) do |c|
+      c.notify = %w[ <%=god_notify_list%> ]
+    end
+  end
+
+  # lifecycle
+  w.lifecycle do |on|
+    on.condition(:flapping) do |c|
+      c.to_state = [:start, :restart]
+      c.times = 5
+      c.within = 5.minute
+      c.transition = :unmonitored
+      c.retry_in = 10.minutes
+      c.retry_times = 5
+      c.retry_within = 2.hours
+      c.notify = %w[ <%=god_notify_list%> ]
+    end
+  end
+end