capistrano-cluster 0.0.10

data/tasks/roles/proxy.rake

@@ -0,0 +1,59 @@
+set :proxy_files, fetch(:proxy_files,{})
+namespace :setup do
+
+  task :proxy do
+
+      on roles(:proxy) do
+
+        install "nginx"
+
+        upload_as :root, file("lb-sysctl.conf"), "/etc/sysctl.d/20-web.conf"
+        sudo :sysctl, "-p", "/etc/sysctl.d/20-web.conf"
+        sudo :mkdir, "-p", "/etc/nginx/servers"
+        upload_as :root, file("nginx/lb-nginx.conf", user: fetch(:user)), "/etc/nginx/nginx.conf", mode: 644
+
+        sudo :rm, "-f", *%w[fastcgi_params proxy_params scgi_params uwsgi_params].map { |file| file.prepend("/etc/nginx/") }
+        sudo :rm, "-rf", *%w[sites-available sites-enabled conf.d].map { |file| file.prepend("/etc/nginx/") }
+
+        sudo "bash -c 'nginx -s reload || service nginx restart'"
+
+      end
+
+  end
+
+end
+
+after "setup:system", "setup:proxy"
+
+namespace :deploy do
+
+  namespace :application do
+
+    task :proxy do
+
+      on roles(:proxy) do
+
+        shared_path = Pathname.new("/etc/nginx/servers/#{fetch(:application)}-#{fetch(:framework_env)}")
+
+        sudo :mkdir, '-p', shared_path
+        sudo :chown, "-R", fetch(:user), shared_path
+
+        if ssl_certificate = fetch(:ssl_certificate)
+          upload_as :root, StringIO.new(ssl_certificate), shared_path.join("ssl.pem")
+        end
+
+        fetch(:proxy_files).each_pair do |file, remote_file|
+          upload_as :root, file(file), shared_path.join(remote_file)
+        end
+
+        upload_as :root, file("nginx/lb-application.conf", ssl: ssl_certificate), shared_path.join("nginx.conf")
+
+        sudo "bash -c 'nginx -s reload || service nginx restart'"
+
+      end
+
+    end
+
+  end
+
+end

data/tasks/roles/rabbit.rake

@@ -0,0 +1,52 @@
+
+namespace :setup do
+
+  before :package_sources, :"rabbitmq:sources"
+
+
+  namespace :rabbitmq do
+    task :sources do
+
+      on roles(:rabbitmq) do
+        unless test "[ -f /etc/apt/sources.list.d/rabbitmq.list ]"
+          execute %q[wget --quiet -O - http://www.rabbitmq.com/rabbitmq-signing-key-public.asc | sudo apt-key add -]
+          upload_as :root, StringIO.new("deb http://www.rabbitmq.com/debian/ testing main"), "/etc/apt/sources.list.d/rabbitmq.list"
+        end
+      end
+
+    end
+  end
+
+  task :rabbitmq do
+
+    on roles(:rabbitmq) do
+      install "rabbitmq-server"
+      upload_as :rabbitmq, StringIO.new("[rabbitmq_management,rabbitmq_management_visualiser,rabbitmq_stomp,rabbitmq_amqp1_0,rabbitmq_mqtt]."), "/etc/rabbitmq/enabled_plugins"
+    end
+
+  end
+
+end
+
+after "setup:system", "setup:rabbitmq"
+
+
+namespace :deploy do
+  task :rabbitmq do
+  end
+end
+
+namespace :firewall do
+  task :rabbitmq do
+    on roles(:rabbitmq) do |server|
+      sudo :ufw, :allow, :in, :epmd
+      sudo :ufw, :allow, :in, :amqp
+      sudo :ufw, :allow, :in, :'25672'
+      sudo :ufw, :allow, :in, :'15672'
+    end
+  end
+end
+
+
+before "deploy:publishing", "deploy:rabbitmq"
+after "setup:firewall", "firewall:rabbitmq"

data/tasks/roles/resque.rake

@@ -0,0 +1,96 @@
+
+namespace :deploy do
+  before :starting, "deploy:resque:deployment_hooks"
+
+  namespace :resque do
+
+    task :deployment_hooks do
+      after 'deploy:starting',  'deploy:resque:stop'
+      after 'deploy:updated',   'deploy:resque:update'
+      after 'deploy:reverted',  'deploy:resque:stop'
+      after 'deploy:published', 'deploy:resque:start'
+    end
+
+    task :stop do
+      on roles(:resque) do
+        service = [fetch(:application), fetch(:stage), 'resque', '*'].join("-")
+        execute :find, "/etc/init.d", "-type f", "-name #{service}", "-exec {} stop \\;"
+      end
+      on primary :resque_scheduler do
+        execute "/etc/init.d/#{fetch(:application)}-scheduler stop || true"
+      end
+
+    end
+
+
+    task update: :stop do
+      on roles(:resque) do |host|
+
+        match = [fetch(:application), fetch(:stage), 'resque', '*'].join("-")
+        sudo :find, "/etc/init.d", "-type f", "-name '#{match}'", "-exec rm -f {} \\;"
+
+        if test "[ -f /etc/init/#{fetch(:application)}-worker.conf ]"
+          sudo "stop '#{fetch(:application)}-worker' || true"
+          sudo "rm -f /etc/init/#{fetch(:application)}-worker*.conf"
+        end
+
+        host.properties.workers.to_i.times do |worker_id|
+
+          service_name = [fetch(:application), fetch(:stage), 'resque', worker_id].join("-")
+          service service_name do
+            user fetch(:user)
+            working_dir release_path
+
+            pid_file shared_path.join("tmp/pids/resque-#{worker_id}.pid")
+            log_file shared_path.join("log/resque-#{worker_id}.log")
+
+            start "bundle exec rake resque:work RAILS_ENV=#{fetch(:framework_env)} 'QUEUE=*' PIDFILE=#{pid_file} INTERVAL=#{fetch(:resque_interval, 0.1)} LOGFILE=#{log_file}"
+            stop "kill -QUIT $(cat '#{pid_file}')"
+
+          end
+        end
+
+      end
+
+      on primary :resque_scheduler do
+
+        service "#{fetch(:application)}-scheduler" do
+          user fetch(:user)
+          working_dir release_path
+
+          pid_file shared_path.join("tmp/pids/resque-scheduler.pid")
+          log_file shared_path.join("log/resque-scheduler.log")
+
+          start "bundle exec rake environment resque:scheduler RAILS_ENV=#{fetch(:framework_env)} 'QUEUE=*' PIDFILE=#{pid_file} RESQUE_SCHEDULER_INTERVAL=#{fetch(:resque_interval, 0.1)} LOGFILE=#{log_file}"
+          stop "kill -QUIT $(cat '#{pid_file}')"
+        end
+
+      end
+    end
+
+
+    task :start do
+      on roles(:resque) do
+
+        host.properties.workers.to_i.times do |worker_id|
+          service_name = [fetch(:application), fetch(:stage), 'resque', worker_id].join("-")
+          sudo "/etc/init.d/#{service_name}", :start
+        end
+      end
+
+      on primary :resque_scheduler do
+        if test "[ -f '/etc/init.d/#{fetch(:application)}-scheduler' ]"
+          sudo "/etc/init.d/#{fetch(:application)}-scheduler", :start
+        end
+      end
+
+    end
+
+    task restart: [:stop, :start]
+
+
+  end
+
+end
+
+

data/tasks/roles/sidekiq.rake

@@ -0,0 +1,42 @@
+namespace :deploy do
+
+  before :starting, "deploy:sidekiq:deployment_hooks"
+
+  namespace :sidekiq do
+    task :deployment_hooks do
+      after 'deploy:starting',  'deploy:sidekiq:stop'
+      after 'deploy:updated',   'deploy:sidekiq:update'
+      after 'deploy:reverted',  'deploy:sidekiq:stop'
+      after 'deploy:published', 'deploy:sidekiq:start'
+    end
+
+    task :update do
+      on roles(:sidekiq) do
+        service "#{fetch(:application)}-sidekiq" do
+          user fetch(:user)
+          working_dir current_path
+          start "bundle exec sidekiq -e '#{fetch(:framework_env)}' -C '#{current_path}/config/sidekiq.yml' -L '#{current_path}/log/sidekiq.log' -P '#{current_path}/tmp/pids/sidekiq.pid' #{fetch(:sidekiq_options)}"
+          stop "bundle exec sidekiqctl stop '#{current_path}/tmp/pids/sidekiq.pid' 10"
+        end
+      end
+    end
+
+
+    task :start do
+      on roles(:sidekiq) do
+        sudo "/etc/init.d/#{fetch(:application)}-sidekiq" , :start
+      end
+    end
+
+    task :stop do
+      on roles(:sidekiq) do
+        sudo "/etc/init.d/#{fetch(:application)}-sidekiq stop || true"
+      end
+    end
+
+    task restart: [:stop, :start]
+
+  end
+
+end
+

data/tasks/roles/web.rake

@@ -0,0 +1,64 @@
+namespace :setup do
+
+  task :web do
+
+    on roles(:web) do
+      install %w[nginx]
+      upload_as :root, file("lb-sysctl.conf"), "/etc/sysctl.d/20-web.conf"
+      sudo :sysctl, "-p", "/etc/sysctl.d/20-web.conf"
+
+      sudo :mkdir, "-p", "/etc/nginx/servers"
+      upload_as :root, file("nginx/nginx.conf", user: fetch(:user)), "/etc/nginx/nginx.conf"
+      sudo :rm, "-f", *%w[fastcgi_params proxy_params scgi_params uwsgi_params].map { |file| file.prepend("/etc/nginx/") }
+      sudo :rm, "-rf", *%w[sites-available sites-enabled conf.d].map { |file| file.prepend("/etc/nginx/") }
+      sudo "nginx -s reload || sudo service nginx restart"
+    end
+
+  end
+
+end
+
+after "setup:customizations", "setup:web"
+
+namespace :deploy do
+
+  namespace :application do
+
+    task :web do
+
+      on roles(:web) do |host|
+
+        upload! file("unicorn.rb", instances: host.properties.instances ), shared_path.join("config/unicorn.rb")
+        upload! file("nginx/application.conf"), shared_path.join("config/nginx.conf")
+
+        service "#{fetch(:application)}-web" do
+          user fetch(:user)
+          working_dir current_path
+          start "bundle exec unicorn_rails -E '#{fetch(:framework_env)}' -l '#{shared_path.join("tmp/sockets/nginx.sock")}' -c '#{shared_path.join("config/unicorn.rb")}'"
+          stop "kill -TERM $(cat '#{shared_path.join("tmp/pids/unicorn.pid")}')"
+          reload "kill -USR2 $(cat '#{shared_path.join("tmp/pids/unicorn.pid")}') || start "
+        end
+
+      end
+    end
+
+
+
+    task :reload do
+      on roles(:web) do
+        sudo "nginx -s reload || sudo service nginx restart"
+      end
+    end
+
+
+
+  end
+
+end
+
+
+
+namespace :deploy do
+  after :restart, "application:reload"
+end
+

data/tasks/setup/firewall.rake

@@ -0,0 +1,53 @@
+namespace :setup do
+
+  task :firewall do
+
+    def ufw(*rule)
+      sudo :ufw, *rule
+    end
+
+    on roles(:all) do
+      upload_as :root, file("ufw"), "/etc/default/ufw"
+      ufw :allow, "in ssh"
+    end
+
+    on roles(:db) do
+      roles(:app).each do |server|
+        ufw :allow, "proto tcp from #{server.hostname} to any port postgresql"
+      end
+    end
+
+    on roles(:indexer) do
+      roles(:app).each do |server|
+        ufw :allow, "proto tcp from #{server.hostname} to any port 8983"
+      end
+    end
+
+    on roles(:cache) do
+      roles(:app).each do |server|
+        ufw :allow, "proto tcp from #{server.hostname} to any port 6379"
+      end
+    end
+
+    on roles(:web) do
+      roles(:proxy).each do |server|
+        ufw :allow, "proto tcp from #{server.hostname} to any port http"
+      end
+    end
+
+
+    on roles(:proxy).each do |server|
+      ufw :allow, "in http"
+      ufw :allow, "in https"
+    end
+
+    on roles(:all) do
+      execute :yes, "| sudo ufw enable"
+    end
+
+  end
+
+end
+
+
+before "setup:finished", "setup:firewall"

data/tasks/setup.rake

@@ -0,0 +1,96 @@
+
+set :default_packages, %w{tmux htop vim-nox software-properties-common curl wget unattended-upgrades bash-completion git lsof ufw ngrep}
+set :packages, fetch(:packages, [])
+
+desc "Bootstraps the servers"
+task prepare: %w{setup:system setup:finished}
+
+
+namespace :setup do
+
+  task system: [:hostname, :swap, :users, :package_sources, :packages, :customizations]
+
+  task :users do
+    on roles(:all) do |host|
+
+      login_as(:root, on: host) do
+        user = fetch(:user)
+
+        unless test "id -u #{user}"
+          execute :useradd, "--home", "/home/#{user}", "-m", "--shell", "/bin/bash", user
+          execute :mkdir, "-p", "#{home_path}/.ssh"
+          execute :chmod, "0700", "#{home_path}/.ssh"
+          execute :chown, "-R", "#{user}:#{user}", "#{home_path}"
+          upload! StringIO.new("#{user} ALL=NOPASSWD:ALL"), "/etc/sudoers.d/#{user}"
+        end
+
+        upload! file('ssh/authorized_keys', keys: fetch(:deploy_keys,[])), "#{home_path}/.ssh/authorized_keys"
+        upload! file('ssh/config', keys: fetch(:deploy_keys,[])), "#{home_path}/.ssh/config"
+        execute :chmod, "0600", "#{home_path}/.ssh/authorized_keys #{home_path}/.ssh/config"
+        execute :chown, "#{user}:#{user}", "#{home_path}/.ssh/authorized_keys #{home_path}/.ssh/config"
+
+      end
+
+    end
+
+  end
+
+  task :finished
+
+  task :hostname do
+
+    on roles(:all) do |host|
+      login_as :root, on: host do
+        unless (hostname = capture(:hostname).strip).empty?
+          upload_as :root, file("etc/hosts", hostname: hostname), "/etc/hosts"
+        end
+      end
+    end
+
+  end
+
+  task :packages do
+
+    on roles(:all) do
+      upload_as :root, file("apt.conf.d/10periodic"), "/etc/apt/apt.conf.d/10periodic"
+      upload_as :root, file("apt.conf.d/50unattended-upgrades"), "/etc/apt/apt.conf.d/50unattended-upgrades"
+      packages = fetch(:default_packages,[]) + fetch(:packages,[])
+      install *packages unless packages.empty?
+    end
+
+  end
+
+  task :customizations do
+    on roles(:all) do
+      upload_as :root, file("tmux.conf"), "/etc/tmux.conf"
+      sudo :chmod, "0644", "/etc/tmux.conf"
+      upload_as :root, file("sshd_config"), "/etc/ssh/sshd_config"
+      upload_as :root, file("issue.net"), "/etc/issue.net"
+      sudo :service, 'ssh', 'reload'
+    end
+  end
+
+  task :swap do
+    on roles(:swap) do
+      if test "[ ! -f /swap ]"
+        sudo "dd if=/dev/zero of=/swap count=$(cat /proc/meminfo | head -1  | awk '{ print $2*2 }') bs=1KB"
+        sudo "mkswap /swap"
+        sudo "swapon /swap"
+        sudo "echo '/swap none            swap    sw              0       0' >> /etc/fstab "
+      end
+    end
+  end
+
+
+  task :package_sources do
+    on roles(:all) do
+      unless test "[ -f /etc/apt/sources.list.d/pgdg.list ]"
+        execute %q[wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -]
+        sudo %q[echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -c | awk '{print $2}'`-pgdg main" >/etc/apt/sources.list.d/pgdg.list]
+      end
+    end
+  end
+
+end
+
+

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification
+name: capistrano-cluster
+version: !ruby/object:Gem::Version
+  version: 0.0.10
+platform: ruby
+authors:
+- Vlad Verestiuc
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capistrano
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capistrano-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capistrano-bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Setup tasks and role additions for capistrano
+email:
+- verestiuc.vlad@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- capistrano-cluster.gemspec
+- files/Procfile.erb
+- files/apt.conf.d/10periodic
+- files/apt.conf.d/50unattended-upgrades
+- files/database.yml.erb
+- files/etc/hosts.erb
+- files/firewall.erb
+- files/issue.net
+- files/lb-sysctl.conf
+- files/nginx/application.conf.erb
+- files/nginx/lb-application.conf.erb
+- files/nginx/lb-nginx.conf.erb
+- files/nginx/nginx.conf.erb
+- files/pg_hba.conf.erb
+- files/postgresql.conf
+- files/redis/redis.conf
+- files/service.erb
+- files/solr/conf/lang/contractions_ca.txt
+- files/solr/conf/lang/contractions_fr.txt
+- files/solr/conf/lang/contractions_ga.txt
+- files/solr/conf/lang/contractions_it.txt
+- files/solr/conf/lang/hyphenations_ga.txt
+- files/solr/conf/lang/stemdict_nl.txt
+- files/solr/conf/lang/stoptags_ja.txt
+- files/solr/conf/lang/stopwords_ar.txt
+- files/solr/conf/lang/stopwords_bg.txt
+- files/solr/conf/lang/stopwords_ca.txt
+- files/solr/conf/lang/stopwords_cz.txt
+- files/solr/conf/lang/stopwords_da.txt
+- files/solr/conf/lang/stopwords_de.txt
+- files/solr/conf/lang/stopwords_el.txt
+- files/solr/conf/lang/stopwords_en.txt
+- files/solr/conf/lang/stopwords_es.txt
+- files/solr/conf/lang/stopwords_eu.txt
+- files/solr/conf/lang/stopwords_fa.txt
+- files/solr/conf/lang/stopwords_fi.txt
+- files/solr/conf/lang/stopwords_fr.txt
+- files/solr/conf/lang/stopwords_ga.txt
+- files/solr/conf/lang/stopwords_gl.txt
+- files/solr/conf/lang/stopwords_hi.txt
+- files/solr/conf/lang/stopwords_hu.txt
+- files/solr/conf/lang/stopwords_hy.txt
+- files/solr/conf/lang/stopwords_id.txt
+- files/solr/conf/lang/stopwords_it.txt
+- files/solr/conf/lang/stopwords_ja.txt
+- files/solr/conf/lang/stopwords_lv.txt
+- files/solr/conf/lang/stopwords_nl.txt
+- files/solr/conf/lang/stopwords_no.txt
+- files/solr/conf/lang/stopwords_pt.txt
+- files/solr/conf/lang/stopwords_ro.txt
+- files/solr/conf/lang/stopwords_ru.txt
+- files/solr/conf/lang/stopwords_sv.txt
+- files/solr/conf/lang/stopwords_th.txt
+- files/solr/conf/lang/stopwords_tr.txt
+- files/solr/conf/lang/userdict_ja.txt
+- files/solr/conf/mapping-FoldToASCII.txt
+- files/solr/conf/schema.xml
+- files/solr/conf/solrconfig.xml
+- files/solr/conf/stopwords.txt
+- files/solr/solr.xml
+- files/ssh/authorized_keys.erb
+- files/ssh/config
+- files/sshd_config
+- files/tmux.conf
+- files/ufw
+- files/unicorn.rb.erb
+- lib/capistrano/cluster.rb
+- lib/capistrano/cluster/application.rb
+- lib/capistrano/cluster/core_ext/object.rb
+- lib/capistrano/cluster/files.rb
+- lib/capistrano/cluster/packages.rb
+- lib/capistrano/cluster/paths.rb
+- lib/capistrano/cluster/service.rb
+- lib/capistrano/cluster/version.rb
+- tasks/deploy.rake
+- tasks/deploy/application.rake
+- tasks/roles/app.rake
+- tasks/roles/cache.rake
+- tasks/roles/db.rake
+- tasks/roles/indexer.rake
+- tasks/roles/proxy.rake
+- tasks/roles/rabbit.rake
+- tasks/roles/resque.rake
+- tasks/roles/sidekiq.rake
+- tasks/roles/web.rake
+- tasks/setup.rake
+- tasks/setup/firewall.rake
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Environment setup automation.
+test_files: []
+has_rdoc: